00000002:00100000:0.0:1788572469.037821:0:261255:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff9c372d6b4a80 x1875440809920128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/400 e 0 to 0 dl 1788572554 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000040:0.0:1788572469.037834:0:261255:0:(mdc_locks.c:907:mdc_finish_enqueue()) lustre-MDT0000-mdc-ffff9c3711155800: layout return lvb 56 transno 0 00010000:00000001:0.0:1788572469.037837:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.037838:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.037839:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.037841:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.037843:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00000002:00010000:0.0:1788572469.037846:0:261255:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572469.037857:0:261255:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d0917c0. 00010000:00000001:0.0:1788572469.037859:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.037860:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.037861:0:261255:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.037863:0:261255:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.037874:0:261255:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572469.037877:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.037878:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.037879:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.037880:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.037882:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00000002:00010000:0.0:1788572469.037886:0:261255:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.037891:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.037892:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.037894:0:261255:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572469.037895:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.037896:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.037897:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.037899:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.037901:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000040:0.0:1788572469.037907:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3708c76500 count: 2 00010000:00000001:0.0:1788572469.037910:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572469.037914:0:261255:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 16 res 8589945623/3568 (0 0) 00010000:00000001:0.0:1788572469.037917:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.037918:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.037919:0:261255:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572469.037922:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc16037b0 00000002:00002000:0.0:1788572469.037924:0:261255:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry= intent=layout status=0 disp=0x1: rc = 0 00000002:00000001:0.0:1788572469.037927:0:261255:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.037930:0:261255:0:(lmv_intent.c:533:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.037933:0:261255:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572469.037936:0:261255:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200002b17:0xdf0:0x0] suppgids 0 0: rc 0 00000080:00000001:0.0:1788572469.037940:0:261255:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.037942:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.037946:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440809920128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/400 e 0 to 0 dl 1788572554 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.037952:0:261255:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572469.037954:0:261255:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572469.037957:0:261255:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572469.037961:0:261255:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572469.037963:0:261255:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572469.037965:0:261255:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=17 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572469.037967:0:261255:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572469.037969:0:261255:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7600. 02000000:00000001:0.0:1788572469.037975:0:261255:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572469.037976:0:261255:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572469.037978:0:261255:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788572469.037981:0:261255:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572469.037982:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:0.0:1788572469.037986:0:261255:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6800. 00000080:00010000:0.0:1788572469.037992:0:261255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdf0:0x0](ffff9c3708705b88) for lock 0x14415860931681cf 00800000:00000001:0.0:1788572469.037996:0:261255:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.037999:0:261255:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.038000:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.038001:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.038003:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.038005:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038007:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038009:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038010:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.038012:0:261255:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.038014:0:261255:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.038017:0:261255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931681cf from it ffffb95bc1603a78 00010000:00000001:0.0:1788572469.038019:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.038021:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.038022:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.038024:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038026:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038028:0:261255:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.038031:0:261255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.038038:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038039:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.038040:0:261255:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.038043:0:261255:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.038045:0:261255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 2/0,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.038052:0:261255:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.038054:0:261255:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.038056:0:261255:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.038072:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038074:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.038076:0:261255:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038079:0:261255:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.038080:0:261255:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038082:0:261255:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038085:0:261255:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038088:0:261255:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.038089:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038090:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.038092:0:261255:0:(file.c:7081:ll_layout_intent()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038095:0:261255:0:(file.c:6135:ll_take_md_lock()) Process entered 00000080:00000040:0.0:1788572469.038097:0:261255:0:(file.c:6137:ll_take_md_lock()) trying to match res [0x200002b17:0xdf0:0x0] 00800000:00000001:0.0:1788572469.038101:0:261255:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.038103:0:261255:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.038105:0:261255:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.038108:0:261255:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.038110:0:261255:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.038112:0:261255:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdf0:0x0] 00800000:00000001:0.0:1788572469.038114:0:261255:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.038117:0:261255:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.038119:0:261255:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.038121:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3708c76500 count: 2 00010000:00000001:0.0:1788572469.038123:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.038126:0:261255:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.038129:0:261255:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572469.038133:0:261255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 2/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572469.038140:0:261255:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.038145:0:261255:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.038148:0:261255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 2/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572469.038155:0:261255:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=16 : 16 : 10) 00800000:00000002:0.0:1788572469.038157:0:261255:0:(lmv_obd.c:4142:lmv_lock_match()) Lock match for [0x200002b17:0xdf0:0x0]: 16 00000080:00000001:0.0:1788572469.038159:0:261255:0:(file.c:6142:ll_take_md_lock()) Process leaving (rc=16 : 16 : 10) 00000080:00000001:0.0:1788572469.038161:0:261255:0:(file.c:6938:ll_layout_lock_set()) Process entered 00010000:00000001:0.0:1788572469.038162:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.038164:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.038165:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.038167:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038169:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00000080:00010000:0.0:1788572469.038173:0:261255:0:(file.c:6948:ll_layout_lock_set()) ### file [0x200002b17:0xdf0:0x0](00000000ac87c900) being reconfigured ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00800000:00000001:0.0:1788572469.038180:0:261255:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.038181:0:261255:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.038182:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.038183:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.038184:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=4 00000020:00000001:0.0:1788572469.038186:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038188:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038190:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038191:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.038192:0:261255:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.038193:0:261255:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038196:0:261255:0:(file.c:6863:ll_layout_fetch()) Process entered 00000080:00000002:0.0:1788572469.038198:0:261255:0:(file.c:6866:ll_layout_fetch()) [0x200002b17:0xdf0:0x0] LVB_READY=0 l_lvb_data=ffff9c372d0917c0 l_lvb_len=56 00000080:00000001:0.0:1788572469.038200:0:261255:0:(file.c:6869:ll_layout_fetch()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038202:0:261255:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572469.038204:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.038209:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887902600 : -109714821649016 : ffff9c3707076d88) 00000020:00001000:0.0:1788572469.038211:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3707076d80 00000020:00000001:0.0:1788572469.038213:0:261255:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.038217:0:261255:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.038230:0:261255:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.038237:0:261255:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116261e0. 00020000:00000010:0.0:1788572469.038241:0:261255:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106f00. 00020000:00000010:0.0:1788572469.038247:0:261255:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626060. 00020000:00000001:0.0:1788572469.038249:0:261255:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061635168 : -109714647916448 : ffff9c3711626060) 00020000:00000002:0.0:1788572469.038251:0:261255:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626060, objid 0xdf0:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.038256:0:261255:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.038260:0:261255:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106f00: ostid: 0x280000401:21129 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572469.038481:0:261255:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3730ca34d8) owner ffff9c370259cf80 00020000:00000001:0.0:1788572469.038484:0:261255:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572469.038487:0:261255:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3730ca34d8) owner ffff9c370259cf80 00020000:00000010:0.0:1788572469.038492:0:261255:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106f00. 00020000:00000010:0.0:1788572469.038496:0:261255:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116261e0. 00020000:00000010:0.0:1788572469.038498:0:261255:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626060. 00020000:00000002:0.0:1788572469.038500:0:261255:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdf0:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572469.038502:0:261255:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038504:0:261255:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038507:0:261255:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572469.038510:0:261255:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca34d8) owner 0000000000000000/ffff9c370259cf80 00020000:00000002:0.0:1788572469.038513:0:261255:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c370438eea0 addref 2/0 by ffff9c370259cf80. 00020000:00000002:0.0:1788572469.038516:0:261255:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca34d8) owner 0000000000000000/ffff9c370259cf80 00020000:00000001:0.0:1788572469.038518:0:261255:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572469.038520:0:261255:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572469.038522:0:261255:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572469.038523:0:261255:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572469.038525:0:261255:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572469.038528:0:261255:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdf0:0x0]: layout version change: 4294967294 -> 0 00000020:00001000:0.0:1788572469.038531:0:261255:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076d80 00000080:00000001:0.0:1788572469.038534:0:261255:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038536:0:261255:0:(file.c:6985:ll_layout_lock_set()) Process leaving 00010000:00000001:0.0:1788572469.038537:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038538:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.038540:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.038541:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.038542:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1aa00 refcount=3 00000020:00000001:0.0:1788572469.038544:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038547:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303809536 : -109714405742080 : ffff9c371fd1aa00) 00010000:00000001:0.0:1788572469.038549:0:261255:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.038552:0:261255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.038558:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038559:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.038560:0:261255:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.038562:0:261255:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.038565:0:261255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1aa00/0x14415860931681cf lrc: 2/0,0 mode: CR/CR res: [0x200002b17:0xdf0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1a2c expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572469.038571:0:261255:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.038572:0:261255:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.038574:0:261255:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.038576:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038577:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.038578:0:261255:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038580:0:261255:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.038581:0:261255:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038583:0:261255:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038585:0:261255:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.038586:0:261255:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.038587:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.038588:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.038589:0:261255:0:(file.c:7026:ll_layout_lock_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038592:0:261255:0:(file.c:7145:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.038595:0:261255:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572469.038600:0:261255:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdf0:0x0]io ffff9c3731326a00 type 3 ignore/verify layout 0/0 00020000:00000002:0.0:1788572469.038603:0:261255:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca34d8) owner 0000000000000000/ffff9c370259cf80 00020000:00000001:0.0:1788572469.038606:0:261255:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572469.038608:0:261255:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572469.038610:0:261255:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572469.038612:0:261255:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038614:0:261255:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572469.038617:0:261255:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572469.038618:0:261255:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038621:0:261255:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.038623:0:261255:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572469.038624:0:261255:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572469.038625:0:261255:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca34d8) owner 0000000000000000/ffff9c370259cf80 00000020:00000001:0.0:1788572469.038627:0:261255:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038631:0:261255:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572469.038633:0:261255:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572469.038664:0:261255:0:(lov_io.c:1293:lov_io_setattr_iter_init()) Process entered 00020000:00000001:0.0:1788572469.038666:0:261255:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572469.038668:0:261255:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572469.038672:0:261255:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572469.038676:0:261255:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572469.038677:0:261255:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572469.038678:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.038680:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887902600 : -109714821649016 : ffff9c3707076d88) 00000020:00001000:0.0:1788572469.038683:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3707076d80 00000020:00000001:0.0:1788572469.038686:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3732214080 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038688:0:261255:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572469.038691:0:261255:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.038692:0:261255:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572469.038693:0:261255:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038695:0:261255:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038697:0:261255:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314384704 : -109714395166912 : ffff9c3720730740) 00000020:00000001:0.0:1788572469.038699:0:261255:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000008:00000001:0.0:1788572469.038701:0:261255:0:(osc_io.c:519:osc_io_iter_init()) Process entered 00000008:00000001:0.0:1788572469.038705:0:261255:0:(osc_io.c:539:osc_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038707:0:261255:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038709:0:261255:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038711:0:261255:0:(lov_io.c:1302:lov_io_setattr_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038713:0:261255:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038715:0:261255:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572469.038717:0:261255:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572469.038719:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572469.038720:0:261255:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572469.038721:0:261255:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572469.038722:0:261255:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038724:0:261255:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038726:0:261255:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038727:0:261255:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038729:0:261255:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572469.038730:0:261255:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038731:0:261255:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038732:0:261255:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:0.0:1788572469.038736:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3732214530 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038738:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3732214530 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038740:0:261255:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572469.038742:0:261255:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038744:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3732214530 -> ffff9c3732214530 00020000:00000001:0.0:1788572469.038746:0:261255:0:(lov_io.c:1857:lov_io_setattr_start()) Process entered 00020000:00000001:0.0:1788572469.038747:0:261255:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572469.038749:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572469.038750:0:261255:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572469.038752:0:261255:0:(osc_io.c:717:osc_io_setattr_start()) Process entered 00000020:00000001:0.0:1788572469.038754:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a700 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038755:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a700 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038757:0:261255:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00000020:00000001:0.0:1788572469.038759:0:261255:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038761:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a700 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038762:0:261255:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00020000:00000001:0.0:1788572469.038765:0:261255:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572469.038766:0:261255:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038768:0:261255:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038769:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a700 -> ffff9c3732214530 00000020:00000001:0.0:1788572469.038771:0:261255:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572469.038773:0:261255:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.038774:0:261255:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.038776:0:261255:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572469.038777:0:261255:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572469.038778:0:261255:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572469.038781:0:261255:0:(osc_request.c:199:osc_setattr_async()) Process entered 00000100:00000010:0.0:1788572469.038786:0:261255:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572469.038790:0:261255:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=48 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572469.038795:0:261255:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.038798:0:261255:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.038801:0:261255:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.038806:0:261255:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6800. 00000100:00000001:0.0:1788572469.038812:0:261255:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.038820:0:261255:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.038825:0:261255:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.038833:0:261255:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4a80 x1875440809920256/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572469.038856:0:261255:0:(osc_request.c:230:osc_setattr_async()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572469.038859:0:261255:0:(osc_io.c:838:osc_io_setattr_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038860:0:261255:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038862:0:261255:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038863:0:261255:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.038864:0:261255:0:(lov_io.c:1872:lov_io_setattr_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038877:0:261255:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.038879:0:261255:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572469.038880:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00020000:00000001:0.0:1788572469.038881:0:261255:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572469.038883:0:261255:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1788572469.038900:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572469.038904:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572469.038906:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.038913:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.038915:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.038918:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.038919:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.038921:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.038926:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440809920256/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.038934:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.038936:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.038940:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.038942:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.038945:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.038948:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809920256:192.168.202.148@tcp:2:touch.0 00000100:00000001:0.0:1788572469.038952:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.038958:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.038960:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.038962:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.038967:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572469.038969:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.038976:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.038980:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.038987:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572469.038994:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809920256, portal 4 00000100:00000001:0.0:1788572469.038996:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.038998:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000040:0.0:1788572469.039005:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4a80 x1875440809920256/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.039018:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.039021:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.039024:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788572469.039027:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 28, xid 1875440809920256, offset 0 00000400:00000200:0.0:1788572469.039033:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.039040:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.039048:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.039053:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.039057:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.041375:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.041379:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.041384:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539594:539594:256:4294967295] 192.168.202.148@tcp LPNI seq info [539559:539559:8:4294967295] 00000400:00000200:0.0:1788572469.041390:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.041401:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.041409:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.041419:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572469.041426:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.041433:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.041437:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 536 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.041448:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.041454:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 174097 usecs (175 jiffies) 00000400:00000200:0.0:1788572469.041458:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.041461:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307608222 00000100:00000001:0.0:1788572469.041464:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.041467:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307608222 00000100:00000001:0.0:1788572469.041469:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.041471:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.041474:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.041479:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.041484:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.049705:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.049710:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.049718:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.049720:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.049724:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572469.049729:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed312a8 00000400:00000010:0.0:1788572469.049732:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed312a8. 00000100:00000001:0.0:1788572469.049735:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.049740:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4a80 x1875440809920256/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.049748:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4a80 x1875440809920256/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.049754:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.049756:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.049759:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.049762:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.049764:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.049779:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.049782:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.049792:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=33 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572469.049796:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=84 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572469.049806:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.049808:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.049811:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.049815:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.049818:0:2403:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572469.049820:0:2403:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x0/0x0] 00000100:00000001:0.0:1788572469.049821:0:2403:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572469.049824:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.049825:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788572469.049829:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.049831:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727829 00000100:00000001:0.0:1788572469.049833:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.049835:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.049841:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440809920256/t38654727877(38654727877) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.049849:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.049851:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.049853:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572469.049856:0:2403:0:(osc_request.c:175:osc_setattr_interpret()) Process entered 00000008:00000001:0.0:1788572469.049885:0:2403:0:(osc_request.c:188:osc_setattr_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.049890:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440809920256/t38654727877(38654727877) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Interpret:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.049898:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809920256:192.168.202.148@tcp:2:touch.0 00000100:00000001:0.0:1788572469.049902:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.049905:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.049908:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440809920256/t38654727877(38654727877) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.049916:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.049918:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.049925:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.049927:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.049929:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.049932:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572469.051192:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.051197:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572469.051202:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.051209:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.051224:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3088 00000400:00000010:0.0:1788572469.051227:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3088. 00000100:00000001:0.0:1788572469.051231:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.051236:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4a80 x1875440809920512/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.051245:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.051248:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4a80 x1875440809920512/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.051255:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.051257:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.062107:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.062115:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572469.062122:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.062131:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.062138:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c16e8 00000400:00000010:0.0:1788572469.062141:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c16e8. 00000100:00000001:0.0:1788572469.062145:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.062152:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c3731d47b80 x1875440809920768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.062163:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.062167:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c3731d47b80 x1875440809920768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.062174:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.062176:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.177607:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.177612:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572469.177616:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.177621:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.177627:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f36e8 00000400:00000010:0.0:1788572469.177629:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f36e8. 00000100:00000001:0.0:1788572469.177633:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.177673:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4700 x1875440809922560/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.177681:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.177685:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4700 x1875440809922560/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.177691:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.177692:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.316031:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.316037:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572469.316042:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.316047:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.316053:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572469.316055:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572469.316059:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.316071:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5500 x1875440809925632/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.316081:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.316085:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5500 x1875440809925632/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.316327:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.316329:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.324334:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.324343:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572469.324351:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.324360:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.324369:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1330 00000400:00000010:0.0:1788572469.324371:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1330. 00000100:00000001:0.0:1788572469.324377:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.324384:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72a00 x1875440809925888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.324394:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.324397:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72a00 x1875440809925888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.324404:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.324405:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000002:00000001:0.0:1788572469.405356:0:261255:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.405359:0:261255:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.405362:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3707e8f900 count: 2 00010000:00000001:0.0:1788572469.405365:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.405368:0:261255:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.405372:0:261255:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572469.405376:0:261255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e908e00/0x1441586093168231 lrc: 2/1,0 mode: CR/CR res: [0x200002b17:0xdf7:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b83 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572469.405383:0:261255:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.405388:0:261255:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.405391:0:261255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e908e00/0x1441586093168231 lrc: 2/1,0 mode: CR/CR res: [0x200002b17:0xdf7:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b83 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572469.405397:0:261255:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=16 : 16 : 10) 00800000:00000002:0.0:1788572469.405399:0:261255:0:(lmv_obd.c:4142:lmv_lock_match()) Lock match for [0x200002b17:0xdf7:0x0]: 16 00000080:00000001:0.0:1788572469.405401:0:261255:0:(file.c:6142:ll_take_md_lock()) Process leaving (rc=16 : 16 : 10) 00000080:00000001:0.0:1788572469.405403:0:261255:0:(file.c:6938:ll_layout_lock_set()) Process entered 00010000:00000001:0.0:1788572469.405405:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.405407:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.405408:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372e908e00 refcount=3 00000020:00000001:0.0:1788572469.405410:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00010000:00000001:0.0:1788572469.405412:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00000080:00010000:0.0:1788572469.405415:0:261255:0:(file.c:6948:ll_layout_lock_set()) ### file [0x200002b17:0xdf7:0x0](00000000f4ca542b) being reconfigured ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e908e00/0x1441586093168231 lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf7:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b83 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00800000:00000001:0.0:1788572469.405433:0:261255:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.405435:0:261255:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.405436:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.405437:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.405439:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372e908e00 refcount=4 00000020:00000001:0.0:1788572469.405441:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00010000:00000001:0.0:1788572469.405443:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00010000:00000001:0.0:1788572469.405445:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.405446:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.405447:0:261255:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.405448:0:261255:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.405451:0:261255:0:(file.c:6863:ll_layout_fetch()) Process entered 00000080:00000002:0.0:1788572469.405452:0:261255:0:(file.c:6866:ll_layout_fetch()) [0x200002b17:0xdf7:0x0] LVB_READY=0 l_lvb_data=ffff9c3707985540 l_lvb_len=56 00000080:00000001:0.0:1788572469.405456:0:261255:0:(file.c:6869:ll_layout_fetch()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.405457:0:261255:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572469.405460:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.405463:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792520 : -109714643759096 : ffff9c3711a1d008) 00000020:00001000:0.0:1788572469.405465:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d000 00000020:00000001:0.0:1788572469.405467:0:261255:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.405470:0:261255:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.405473:0:261255:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.405479:0:261255:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626060. 00020000:00000010:0.0:1788572469.405483:0:261255:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106f00. 00020000:00000010:0.0:1788572469.405488:0:261255:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116261e0. 00020000:00000001:0.0:1788572469.405490:0:261255:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061635552 : -109714647916064 : ffff9c37116261e0) 00020000:00000002:0.0:1788572469.405493:0:261255:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116261e0, objid 0xdf7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.405497:0:261255:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.405501:0:261255:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106f00: ostid: 0x2c0000401:21018 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572469.405505:0:261255:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711cba1f0) owner ffff9c370259cf80 00020000:00000001:0.0:1788572469.405508:0:261255:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572469.405510:0:261255:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711cba1f0) owner ffff9c370259cf80 00020000:00000010:0.0:1788572469.405514:0:261255:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106f00. 00020000:00000010:0.0:1788572469.405517:0:261255:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626060. 00020000:00000010:0.0:1788572469.405520:0:261255:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116261e0. 00020000:00000002:0.0:1788572469.405521:0:261255:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdf7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572469.405524:0:261255:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405525:0:261255:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405528:0:261255:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572469.405531:0:261255:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cba1f0) owner 0000000000000000/ffff9c370259cf80 00020000:00000002:0.0:1788572469.405534:0:261255:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c37025a4000 addref 2/0 by ffff9c370259cf80. 00020000:00000002:0.0:1788572469.405536:0:261255:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cba1f0) owner 0000000000000000/ffff9c370259cf80 00020000:00000001:0.0:1788572469.405538:0:261255:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572469.405540:0:261255:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572469.405541:0:261255:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572469.405542:0:261255:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572469.405544:0:261255:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572469.405547:0:261255:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdf7:0x0]: layout version change: 4294967294 -> 0 00000020:00001000:0.0:1788572469.405550:0:261255:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d000 00000080:00000001:0.0:1788572469.405564:0:261255:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.405566:0:261255:0:(file.c:6985:ll_layout_lock_set()) Process leaving 00010000:00000001:0.0:1788572469.405567:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.405568:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.405569:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.405570:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.405572:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372e908e00 refcount=3 00000020:00000001:0.0:1788572469.405573:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00010000:00000001:0.0:1788572469.405575:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359551200768 : -109714158350848 : ffff9c372e908e00) 00010000:00000001:0.0:1788572469.405578:0:261255:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.405580:0:261255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e908e00/0x1441586093168231 lrc: 3/1,0 mode: CR/CR res: [0x200002b17:0xdf7:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1b83 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.405587:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.405596:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.405598:0:261255:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.405599:0:261255:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.405602:0:261255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e908e00/0x1441586093168231 lrc: 2/0,0 mode: CR/CR res: [0x200002b17:0xdf7:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1b83 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572469.405607:0:261255:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.405609:0:261255:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.405610:0:261255:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.405612:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.405613:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.405614:0:261255:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.405616:0:261255:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.405618:0:261255:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.405619:0:261255:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.405622:0:261255:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.405623:0:261255:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.405625:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.405625:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.405627:0:261255:0:(file.c:7026:ll_layout_lock_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.405630:0:261255:0:(file.c:7145:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.405632:0:261255:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572469.405667:0:261255:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdf7:0x0]io ffff9c371ff509a0 type 3 ignore/verify layout 0/0 00020000:00000002:0.0:1788572469.405671:0:261255:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cba1f0) owner 0000000000000000/ffff9c370259cf80 00020000:00000001:0.0:1788572469.405673:0:261255:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572469.405675:0:261255:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572469.405677:0:261255:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572469.405678:0:261255:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405681:0:261255:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572469.405683:0:261255:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572469.405684:0:261255:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405687:0:261255:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.405688:0:261255:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572469.405689:0:261255:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572469.405691:0:261255:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cba1f0) owner 0000000000000000/ffff9c370259cf80 00000020:00000001:0.0:1788572469.405693:0:261255:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405696:0:261255:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572469.405697:0:261255:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572469.405700:0:261255:0:(lov_io.c:1293:lov_io_setattr_iter_init()) Process entered 00020000:00000001:0.0:1788572469.405702:0:261255:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572469.405704:0:261255:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572469.405707:0:261255:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572469.405710:0:261255:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572469.405711:0:261255:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572469.405713:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.405715:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792520 : -109714643759096 : ffff9c3711a1d008) 00000020:00001000:0.0:1788572469.405717:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d000 00000020:00000001:0.0:1788572469.405720:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3718eeb850 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405722:0:261255:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572469.405725:0:261255:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.405726:0:261255:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572469.405727:0:261255:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405729:0:261255:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405731:0:261255:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293066168 : -109714416485448 : ffff9c371f2dbbb8) 00000020:00000001:0.0:1788572469.405734:0:261255:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000008:00000001:0.0:1788572469.405736:0:261255:0:(osc_io.c:519:osc_io_iter_init()) Process entered 00000008:00000001:0.0:1788572469.405739:0:261255:0:(osc_io.c:539:osc_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405740:0:261255:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405742:0:261255:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405744:0:261255:0:(lov_io.c:1302:lov_io_setattr_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405746:0:261255:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405748:0:261255:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572469.405750:0:261255:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572469.405751:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572469.405752:0:261255:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572469.405754:0:261255:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572469.405755:0:261255:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405757:0:261255:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405759:0:261255:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405760:0:261255:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405762:0:261255:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572469.405763:0:261255:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405764:0:261255:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405766:0:261255:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:0.0:1788572469.405769:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebd00 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405770:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebd00 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405772:0:261255:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572469.405774:0:261255:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405776:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebd00 -> ffff9c3718eebd00 00020000:00000001:0.0:1788572469.405778:0:261255:0:(lov_io.c:1857:lov_io_setattr_start()) Process entered 00020000:00000001:0.0:1788572469.405779:0:261255:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572469.405780:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572469.405781:0:261255:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572469.405784:0:261255:0:(osc_io.c:717:osc_io_setattr_start()) Process entered 00000020:00000001:0.0:1788572469.405786:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405788:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405789:0:261255:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00000020:00000001:0.0:1788572469.405791:0:261255:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405793:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405795:0:261255:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00020000:00000001:0.0:1788572469.405797:0:261255:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572469.405799:0:261255:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405801:0:261255:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405812:0:261255:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572469.405815:0:261255:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572469.405817:0:261255:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.405818:0:261255:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.405819:0:261255:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572469.405820:0:261255:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572469.405822:0:261255:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572469.405825:0:261255:0:(osc_request.c:199:osc_setattr_async()) Process entered 00000100:00000010:0.0:1788572469.405831:0:261255:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572469.405835:0:261255:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000100:00000001:0.0:1788572469.405839:0:261255:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.405842:0:261255:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.405845:0:261255:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.405851:0:261255:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7600. 00000100:00000001:0.0:1788572469.405857:0:261255:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.405864:0:261255:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.405877:0:261255:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.405885:0:261255:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6a00 x1875440809927424/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572469.405905:0:261255:0:(osc_request.c:230:osc_setattr_async()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572469.405908:0:261255:0:(osc_io.c:838:osc_io_setattr_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405910:0:261255:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405911:0:261255:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405913:0:261255:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.405915:0:261255:0:(lov_io.c:1872:lov_io_setattr_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405916:0:261255:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.405919:0:261255:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572469.405920:0:261255:0:(lov_io.c:1312:lov_io_call()) Process entered 00020000:00000001:0.0:1788572469.405922:0:261255:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572469.405923:0:261255:0:(cl_io.c:555:cl_io_end()) Process entered 00000800:00000200:0.0:1788572469.406916:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.406920:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572469.406925:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.406930:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.406936:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788572469.406938:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788572469.406942:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.406946:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6a00 x1875440809927424/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.406956:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.406959:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440809927424/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.406971:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.406973:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572469.411921:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.411927:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.411934:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.411936:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.411940:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2898. 00000400:00000200:0.0:1788572469.411945:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572469.411948:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572469.411951:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.411957:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6a00 x1875440809927424/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.411966:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6a00 x1875440809927424/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.411973:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.411975:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.411978:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.411981:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.411983:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.411985:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.411988:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.411996:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=33 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572469.412000:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=6 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572469.412003:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.412005:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.412007:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.412011:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.412014:0:2403:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572469.412015:0:2403:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x0/0x0] 00000100:00000001:0.0:1788572469.412017:0:2403:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572469.412019:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.412021:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788572469.412024:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.412026:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572469.412028:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.412030:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.412035:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440809927424/t4295010073(4295010073) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.412043:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.412045:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.412047:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572469.412050:0:2403:0:(osc_request.c:175:osc_setattr_interpret()) Process entered 00000008:00000001:0.0:1788572469.412077:0:2403:0:(osc_request.c:188:osc_setattr_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.412082:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440809927424/t4295010073(4295010073) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Interpret:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.412155:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809927424:192.168.202.148@tcp:2:touch.0 00000100:00000001:0.0:1788572469.412160:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.412163:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.412167:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440809927424/t4295010073(4295010073) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.412173:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.412175:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.412183:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.412185:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.412187:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.412189:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572469.415332:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.415338:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572469.415344:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.415349:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.415355:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572469.415358:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572469.415362:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.415368:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5180 x1875440809927680/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.415378:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.415382:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5180 x1875440809927680/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.415389:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.415391:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572469.419893:0:261255:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.419901:0:261255:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.419903:0:261255:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.419906:0:261255:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2f78. 00000400:00000200:0.0:1788572469.419911:0:261255:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572469.419914:0:261255:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572469.419916:0:261255:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.419921:0:261255:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440809927680/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.419930:0:261255:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440809927680/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.419936:0:261255:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.419938:0:261255:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.419941:0:261255:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.419944:0:261255:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.419947:0:261255:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.419950:0:261255:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.419952:0:261255:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.419964:0:261255:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce4f0 time=33 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572469.419968:0:261255:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=65 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572469.419971:0:261255:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.419973:0:261255:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.419976:0:261255:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.419980:0:261255:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.419983:0:261255:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572469.419985:0:261255:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x0/0x0] 00000100:00000001:0.0:1788572469.419987:0:261255:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572469.419989:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.419991:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000001:0.0:1788572469.419994:0:261255:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.419996:0:261255:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572469.420000:0:261255:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.420002:0:261255:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.420007:0:261255:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440809927680/t25769859793(25769859793) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572485 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.420014:0:261255:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.420018:0:261255:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.420020:0:261255:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572469.420024:0:261255:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440809927680/t25769859793(25769859793) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572485 ref 3 fl Interpret:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.420031:0:261255:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:261255:1875440809927680:192.168.202.148@tcp:35:touch.0 00000100:00000001:0.0:1788572469.420037:0:261255:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.420039:0:261255:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.420042:0:261255:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572469.420044:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.420047:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5180 x1875440809927680/t25769859793(25769859793) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572485 ref 3 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.420054:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572469.420057:0:261255:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c372d946a80. 00000100:00000001:0.0:1788572469.420190:0:261255:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572469.420193:0:261255:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.420201:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.420203:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000002:00000001:0.0:1788572469.420206:0:261255:0:(mdc_request.c:1023:mdc_close()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.420209:0:261255:0:(lmv_obd.c:1678:lmv_close()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572469.420215:0:261255:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711214600. 00000080:00000001:0.0:1788572469.420219:0:261255:0:(file.c:268:ll_close_inode_openhandle()) Process leaving 00800000:00000001:0.0:1788572469.420291:0:261255:0:(lmv_obd.c:4201:lmv_clear_open_replay_data()) Process entered 00800000:00000001:0.0:1788572469.420294:0:261255:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.420296:0:261255:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.420299:0:261255:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.420301:0:261255:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.420303:0:261255:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.420305:0:261255:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdf7:0x0] 00800000:00000001:0.0:1788572469.420307:0:261255:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.420312:0:261255:0:(mdc_request.c:844:mdc_clear_open_replay_data()) Process entered 00000002:00100000:0.0:1788572469.420317:0:261255:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9c371ec70700 x1875440809926912/t25769859791(25769859791) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/664 e 0 to 0 dl 1788572554 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572469.420324:0:261255:0:(mdc_request.c:866:mdc_clear_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.420326:0:261255:0:(lmv_obd.c:4207:lmv_clear_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572469.420328:0:261255:0:(file.c:273:ll_close_inode_openhandle()) kfreed 'och': 56 at ffff9c3707985fc0. 00000100:00000001:0.0:1788572469.420331:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.420333:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5180 x1875440809927680/t25769859793(25769859793) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572485 ref 3 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.420339:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.420341:0:261255:0:(file.c:326:ll_md_real_close()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572469.420345:0:261255:0:(file.c:83:ll_file_data_put()) slab-freed '(lfd)': 288 at ffff9c3702c2ec80. 00000080:00000001:0.0:1788572469.420348:0:261255:0:(file.c:438:ll_md_close()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.420354:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603df0 00000080:00000008:0.0:1788572469.420356:0:261255:0:(file.c:523:ll_file_release()) COMPLETED file f434.sanity-099:[0x200002b17:0xdf7:0x0](ffff9c37020e2a08), flags 104001, rc = 0 00000080:00000001:0.0:1788572469.420359:0:261255:0:(file.c:525:ll_file_release()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.420364:0:261255:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572469.420366:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603e00 00000080:00002000:0.0:1788572469.420368:0:261255:0:(dcache.c:127:ll_ddelete()) deleting dentry f434.sanity-099 (ffff9c3719d93438, parent ffff9c3719d93af8, inode ffff9c37020e2a08) hashed, 00000080:00000001:0.0:1788572469.420371:0:261255:0:(dcache.c:130:ll_ddelete()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.420377:0:261255:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:0.0:1788572469.420382:0:261255:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:0.0:1788572469.420384:0:261255:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572469.420385:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603e00 00000080:00002000:0.0:1788572469.420387:0:261255:0:(dcache.c:127:ll_ddelete()) keeping dentry d434.sanity (ffff9c3719d93af8, parent ffff9c370834daf8, inode ffff9c37087b9148) hashed, 00000080:00000001:0.0:1788572469.420389:0:261255:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.420502:0:261255:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572469.420506:0:261255:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572469.420508:0:261255:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572469.420510:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603bc0 00000080:00200000:0.0:1788572469.420512:0:261255:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572469.420518:0:261255:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572469.420524:0:261255:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572469.420529:0:261255:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572469.420533:0:261255:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.420535:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603ae0 00800000:00000002:0.0:1788572469.420537:0:261255:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572469.420541:0:261255:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572469.420543:0:261255:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.420544:0:261255:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.420545:0:261255:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.420546:0:261255:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.420548:0:261255:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.420550:0:261255:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572469.420552:0:261255:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.420554:0:261255:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572469.420559:0:261255:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.420560:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc16039b0 00000002:00010000:0.0:1788572469.420562:0:261255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572469.420566:0:261255:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572469.420568:0:261255:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.420571:0:261255:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.420577:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572469.420580:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.420584:0:261255:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.420588:0:261255:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572469.420834:0:261255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 99 lru_type: 1 02000000:00000001:0.0:1788572469.420846:0:261255:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.420851:0:261255:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.420855:0:261255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 99 lru_type: 1 00000002:00000001:0.0:1788572469.420862:0:261255:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572469.420864:0:261255:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572469.420866:0:261255:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.420878:0:261255:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.420880:0:261255:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572469.420883:0:261255:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572469.420885:0:261255:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572469.420888:0:261255:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572469.420891:0:261255:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572469.420897:0:261255:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572469.420899:0:261255:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.420901:0:261255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572469.420905:0:261255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572469.420908:0:261255:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.420910:0:261255:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.420912:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.420914:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.420917:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572469.420919:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.420922:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.420924:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.420926:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.420927:0:261255:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.420929:0:261255:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.420931:0:261255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1603bf8 00010000:00000001:0.0:1788572469.420933:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.420935:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.420936:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572469.420938:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.420940:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.420942:0:261255:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.420945:0:261255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 99 lru_type: 1 00010000:00000001:0.0:1788572469.420952:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.420953:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.420955:0:261255:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.420957:0:261255:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.420960:0:261255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 100 lru_type: 1 00010000:00000001:0.0:1788572469.420966:0:261255:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.420969:0:261255:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.420972:0:261255:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.420976:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.420977:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.420979:0:261255:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.420982:0:261255:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.420984:0:261255:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.420986:0:261255:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.420989:0:261255:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.420991:0:261255:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.420992:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.420993:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572469.420995:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572469.420997:0:261255:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572469.420999:0:261255:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572469.421003:0:261255:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.421007:0:261255:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572469.421009:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603b88 00000080:00002000:0.0:1788572469.421011:0:261255:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 0 00000080:00000001:0.0:1788572469.421013:0:261255:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.421015:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603bd0 00000080:00200000:0.0:1788572469.421017:0:261255:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572469.421022:0:261255:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572469.421026:0:261255:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572469.421027:0:261255:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572469.421030:0:261255:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572469.421041:0:261255:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f8e0. 00000080:00000001:0.0:1788572469.421045:0:261255:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572469.421046:0:261255:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 3 00000080:00000001:0.0:1788572469.421048:0:261255:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.421052:0:261255:0:(namei.c:1405:ll_atomic_open()) Process entered 00000020:00000001:0.0:1788572469.421053:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603b58 00000080:00200000:0.0:1788572469.421055:0:261255:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f434.sanity-100, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), file ffff9c372ee8c780, open_flags 8941, mode 81b6 opened 0 00000080:00000010:0.0:1788572469.421059:0:261255:0:(namei.c:1432:ll_atomic_open()) kmalloced '(it)': 72 at ffff9c37116261e0. 00000080:00000001:0.0:1788572469.421073:0:261255:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572469.421075:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603a08 00000080:00200000:0.0:1788572469.421077:0:261255:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-100, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=open|creat 00000080:00000010:0.0:1788572469.421083:0:261255:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572469.421086:0:261255:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.421096:0:261255:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572469.421098:0:261255:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.421099:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603928 00800000:00000002:0.0:1788572469.421101:0:261255:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'open|creat' for [0x0:0x0:0x0] 'f434.sanity-100' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.421105:0:261255:0:(lmv_intent.c:299:lmv_intent_open()) Process entered 00800000:00000001:0.0:1788572469.421107:0:261255:0:(lmv_obd.c:2248:lmv_locate_tgt_create()) Process entered 00800000:00000001:0.0:1788572469.421110:0:261255:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.421112:0:261255:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.421113:0:261255:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.421115:0:261255:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.421117:0:261255:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.421119:0:261255:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.421121:0:261255:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.421123:0:261255:0:(lmv_obd.c:2294:lmv_locate_tgt_create()) Process leaving (rc=18446634359068665856 : -109714640885760 : ffff9c3711cda800) 00800000:00000001:0.0:1788572469.421126:0:261255:0:(lmv_obd.c:1208:lmv_fid_alloc()) Process entered 00800000:00000001:0.0:1788572469.421128:0:261255:0:(obd_class.h:931:obd_fid_alloc()) Process entered 00000002:00000001:0.0:1788572469.421130:0:261255:0:(mdc_request.c:2962:mdc_fid_alloc()) Process entered 40000000:00000001:0.0:1788572469.421133:0:261255:0:(fid_request.c:299:seq_client_alloc_fid()) Process entered 40000000:00000040:0.0:1788572469.421135:0:261255:0:(fid_request.c:336:seq_client_alloc_fid()) cli-cli-lustre-MDT0000-mdc-ffff9c3711155800: Allocated FID [0x200002b17:0xdf8:0x0] 40000000:00000001:0.0:1788572469.421138:0:261255:0:(fid_request.c:340:seq_client_alloc_fid()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.421140:0:261255:0:(mdc_request.c:2968:mdc_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.421141:0:261255:0:(obd_class.h:944:obd_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.421143:0:261255:0:(lmv_obd.c:1232:lmv_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.421145:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc16038a8 00800000:00000002:0.0:1788572469.421147:0:261255:0:(lmv_intent.c:382:lmv_intent_open()) OPEN_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdf8:0x0], name='f434.sanity-100' -> mds #0 layout version 1 00000002:00000001:0.0:1788572469.421150:0:261255:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.421152:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc16037e0 00000002:00010000:0.0:1788572469.421154:0:261255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-100,[0x200002b17:0xdf8:0x0]) in obj [0x200002b17:0xd94:0x0], intent: open|creat flags 0104502 00000002:00000001:0.0:1788572469.421157:0:261255:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572469.421160:0:261255:0:(mdc_locks.c:258:mdc_intent_open_pack()) Process entered 00000002:00000001:0.0:1788572469.421163:0:261255:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00000002:00000001:0.0:1788572469.421167:0:261255:0:(mdc_reint.c:67:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.421168:0:261255:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572469.421171:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572469.421173:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.421176:0:261255:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572469.421179:0:261255:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.421181:0:261255:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.421182:0:261255:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572469.421188:0:261255:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572469.421191:0:261255:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=40 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572469.421197:0:261255:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572469.421199:0:261255:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.421201:0:261255:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572469.421205:0:261255:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.421208:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.421209:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.421211:0:261255:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.421214:0:261255:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.421216:0:261255:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.421218:0:261255:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.421357:0:261255:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572469.421364:0:261255:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.421369:0:261255:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.421371:0:261255:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.421372:0:261255:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.421374:0:261255:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.421376:0:261255:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.421384:0:261255:0:(mdc_locks.c:430:mdc_intent_open_pack()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00010000:00000001:0.0:1788572469.421387:0:261255:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572469.421390:0:261255:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572469.421393:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572469.421395:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.421397:0:261255:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572469.421401:0:261255:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c371fd1a200. 00000020:00000001:0.0:1788572469.421404:0:261255:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572469.421407:0:261255:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c371fd1a200 with handle 0x1441586093168238 to hash 00000020:00000001:0.0:1788572469.421408:0:261255:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572469.421410:0:261255:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.421413:0:261255:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.421416:0:261255:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.421430:0:261255:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.421433:0:261255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261255 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.421442:0:261255:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261255 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788572469.421453:0:261255:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788572469.421461:0:261255:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b5500 x1875440809927936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788572469.421470:0:261255:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261255 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572469.421475:0:261255:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572469.421477:0:261255:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572469.421483:0:261255:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5480. 00000100:00000001:0.0:1788572469.421485:0:261255:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288714368 : -109714420837248 : ffff9c371eeb5480) 00000100:00000001:0.0:1788572469.421488:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.421489:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788572469.421493:0:261255:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.421496:0:261255:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.421498:0:261255:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572469.421499:0:261255:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.421504:0:261255:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440809927936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 0 ref 2 fl New:PQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.421511:0:261255:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.421513:0:261255:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.421517:0:261255:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.421519:0:261255:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.421522:0:261255:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.421525:0:261255:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:261255:1875440809927936:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572469.421529:0:261255:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.421534:0:261255:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.421536:0:261255:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.421538:0:261255:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.421604:0:261255:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572469.421607:0:261255:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.421615:0:261255:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.421620:0:261255:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.421627:0:261255:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff9c372d20f000. 00000100:00000200:0.0:1788572469.421668:0:261255:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1875440809927936, portal 10 00000100:00000001:0.0:1788572469.421670:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.421672:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000040:0.0:1788572469.421679:0:261255:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5500 x1875440809927936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.421685:0:261255:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.421687:0:261255:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.421691:0:261255:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572469.421694:0:261255:0:(niobuf.c:94:ptl_send_buf()) Sending 920 bytes to portal 12, xid 1875440809927936, offset 0 00000400:00000200:0.0:1788572469.421700:0:261255:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.421707:0:261255:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.421714:0:261255:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.421719:0:261255:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.421723:0:261255:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.421726:0:261255:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.421729:0:261255:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.421733:0:261255:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539631:539631:256:4294967295] 192.168.202.148@tcp LPNI seq info [539596:539596:8:4294967295] 00000400:00000200:0.0:1788572469.421738:0:261255:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.421747:0:261255:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.421752:0:261255:0:(socklnd_cb.c:931:ksocknal_send()) sending 920 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.421758:0:261255:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572469.421765:0:261255:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.421772:0:261255:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.421777:0:261255:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 1016 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.421786:0:261255:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.421790:0:261255:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14917 usecs (15 jiffies) 00000400:00000200:0.0:1788572469.421793:0:261255:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.421796:0:261255:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307608443 00000100:00000001:0.0:1788572469.421799:0:261255:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.421801:0:261255:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307608443 00000100:00000001:0.0:1788572469.421803:0:261255:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.421806:0:261255:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.421809:0:261255:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.421811:0:261255:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.421813:0:261255:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572469.421816:0:261255:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5480 going to sleep for 80 seconds 00000100:00000001:0.0:1788572469.421818:0:261255:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.421821:0:261255:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.436886:0:261255:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.436955:0:261255:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.436957:0:261255:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.436960:0:261255:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572469.436965:0:261255:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372d20f000 00000400:00000010:0.0:1788572469.436968:0:261255:0:(lib-lnet.h:286:lnet_md_free()) kfreed 'md': 632 at ffff9c372d20f000. 00000100:00000001:0.0:1788572469.436971:0:261255:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.436975:0:261255:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5500 x1875440809927936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.436989:0:261255:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5500 x1875440809927936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/66584 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RPQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.436995:0:261255:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.436997:0:261255:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.436999:0:261255:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.437003:0:261255:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.437005:0:261255:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.437007:0:261255:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.437011:0:261255:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.437020:0:261255:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=33 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572469.437024:0:261255:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=65 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572469.437027:0:261255:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.437029:0:261255:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.437031:0:261255:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.437035:0:261255:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.437038:0:261255:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572469.437039:0:261255:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x60000d9a7/0x1] 00000100:00000001:0.0:1788572469.437041:0:261255:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572469.437043:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.437045:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788572469.437048:0:261255:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.437050:0:261255:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572469.437052:0:261255:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.437054:0:261255:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.437059:0:261255:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/664 e 0 to 0 dl 1788572554 ref 3 fl Rpc:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.437174:0:261255:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.437177:0:261255:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.437179:0:261255:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572469.437183:0:261255:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/664 e 0 to 0 dl 1788572554 ref 3 fl Interpret:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.437189:0:261255:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:261255:1875440809927936:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572469.437195:0:261255:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.437197:0:261255:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.437200:0:261255:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572469.437202:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.437205:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/664 e 0 to 0 dl 1788572554 ref 3 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.437211:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572469.437214:0:261255:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5480. 00000100:00000001:0.0:1788572469.437217:0:261255:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572469.437218:0:261255:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.437278:0:261255:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572469.437283:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437286:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437290:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=4 00000020:00000001:0.0:1788572469.437293:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437296:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000040:0.0:1788572469.437303:0:261255:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c371fd1a200, remote cookie: 0xd9894672d99a1b9f, flags: 0x1001 00010000:00010000:0.0:1788572469.437309:0:261255:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/0,1 mode: --/CW res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572469.437318:0:261255:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdf8:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572469.437321:0:261255:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572469.437330:0:261255:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76700. 00010000:00000010:0.0:1788572469.437335:0:261255:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676300. 00010000:00000001:0.0:1788572469.437340:0:261255:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.437343:0:261255:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/0,1 mode: --/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.437352:0:261255:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/0,1 mode: --/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437364:0:261255:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572469.437368:0:261255:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572469.437370:0:261255:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572469.437371:0:261255:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572469.437373:0:261255:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572469.437375:0:261255:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdf8:0x0].0x0 (ffff9c3708c76700) refcount = 1 00010000:00010000:0.0:1788572469.437379:0:261255:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/0,1 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437386:0:261255:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572469.437389:0:261255:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572469.437391:0:261255:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.437394:0:261255:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572469.437396:0:261255:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.437399:0:261255:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/0,1 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437405:0:261255:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572469.437407:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437409:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.437410:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437411:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.437412:0:261255:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.437490:0:261255:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572469.437494:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437495:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437499:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.437501:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437503:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437506:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437506:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437508:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=4 00000020:00000001:0.0:1788572469.437509:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437511:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437514:0:261255:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.437517:0:261255:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.437521:0:261255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 5/1,1 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437527:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437528:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.437529:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437530:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437531:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=5 00000020:00000001:0.0:1788572469.437533:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437535:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437537:0:261255:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.437539:0:261255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 5/1,1 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437545:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437546:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572469.437549:0:261255:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437746:0:261255:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.437747:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437748:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.437750:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437751:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572469.437758:0:261255:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/664 e 0 to 0 dl 1788572554 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572469.437767:0:261255:0:(mdc_request.c:760:mdc_set_open_replay_data()) Process entered 00000002:00100000:0.0:1788572469.437771:0:261255:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 920/664 e 0 to 0 dl 1788572554 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572469.437783:0:261255:0:(mdc_request.c:812:mdc_set_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.437790:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437791:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437793:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.437795:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437797:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00000002:00010000:0.0:1788572469.437801:0:261255:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572469.437811:0:261255:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091d00. 00010000:00000001:0.0:1788572469.437814:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437815:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.437816:0:261255:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.437818:0:261255:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.437820:0:261255:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00000100:00000001:0.0:1788572469.437822:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.437824:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788572469.437826:0:261255:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.437827:0:261255:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00010000:00000001:0.0:1788572469.437829:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437830:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437831:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.437832:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437834:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00000002:00010000:0.0:1788572469.437838:0:261255:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a200/0x1441586093168238 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1b9f expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.437844:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437845:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.437847:0:261255:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572469.437849:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.437850:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.437852:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.437853:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.437855:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000040:0.0:1788572469.437860:0:261255:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3708c76700 count: 2 00010000:00000001:0.0:1788572469.437863:0:261255:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572469.437866:0:261255:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3576 (0 0) 00010000:00000001:0.0:1788572469.437880:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.437881:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.437882:0:261255:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572469.437885:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603768 00000002:00002000:0.0:1788572469.437888:0:261255:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-100 intent=open|creat status=0 disp=0x1800037: rc = 0 00000002:00000001:0.0:1788572469.437891:0:261255:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.437895:0:261255:0:(lmv_intent.c:429:lmv_intent_open()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.437897:0:261255:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572469.437900:0:261255:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572469.437903:0:261255:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.437907:0:261255:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.437910:0:261255:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572469.437912:0:261255:0:(namei.c:674:ll_lookup_it_finish()) it ffff9c37116261e0 it_disposition 1800037 00000080:00002000:0.0:1788572469.437918:0:261255:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c3719f64000 inode 0000000000000000 refc 1 flags 0xe 00000080:00000001:0.0:1788572469.437923:0:261255:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572469.437926:0:261255:0:(namei.c:1624:ll_create_it()) Process entered 00000020:00000001:0.0:1788572469.437928:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603990 00000080:00200000:0.0:1788572469.437929:0:261255:0:(namei.c:1628:ll_create_it()) VFS Op:name=f434.sanity-100, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=open|creat 00000080:00000001:0.0:1788572469.437932:0:261255:0:(namei.c:1575:ll_create_node()) Process entered 00000080:00000001:0.0:1788572469.437934:0:261255:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572469.437938:0:261255:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572469.437941:0:261255:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572469.437944:0:261255:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329144 : 144115373145329144 : 200002b17000df8) 00000080:00000001:0.0:1788572469.437946:0:261255:0:(namei.c:113:ll_iget()) Process entered 00000080:00000010:0.0:1788572469.437959:0:261255:0:(super25.c:42:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff9c3708707380. 00000020:00000001:0.0:1788572469.437968:0:261255:0:(class_obd.c:978:obd_heat_clear()) Process entered 00000020:00000001:0.0:1788572469.437970:0:261255:0:(class_obd.c:981:obd_heat_clear()) Process leaving 00000080:00000001:0.0:1788572469.437972:0:261255:0:(llite_lib.c:3353:ll_read_inode2()) Process entered 00000080:00200000:0.0:1788572469.437975:0:261255:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200002b17:0xdf8:0x0](ffff9c3708707448) 00000020:00000001:0.0:1788572469.437978:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.437982:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792808 : -109714643758808 : ffff9c3711a1d128) 00000020:00001000:0.0:1788572469.437985:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d120 00000020:00000001:0.0:1788572469.437988:0:261255:0:(lu_object.c:830:lu_object_find_at()) Process entered 00000080:00000010:0.0:1788572469.437993:0:261255:0:(vvp_object.c:304:vvp_object_alloc()) slab-alloced 'vob': 200 at ffff9c3732214af0. 00000020:00000001:0.0:1788572469.437995:0:261255:0:(cl_object.c:60:cl_object_header_init()) Process entered 00000020:00000001:0.0:1788572469.437997:0:261255:0:(cl_object.c:68:cl_object_header_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.438003:0:261255:0:(lov_object.c:2359:lov_object_alloc()) Process entered 00020000:00000010:0.0:1788572469.438007:0:261255:0:(lov_object.c:2360:lov_object_alloc()) slab-alloced 'lov': 248 at ffff9c3730ca36c8. 00020000:00000001:0.0:1788572469.438009:0:261255:0:(lov_object.c:2374:lov_object_alloc()) Process leaving (rc=18446634359588533960 : -109714121017656 : ffff9c3730ca36c8) 00020000:00000001:0.0:1788572469.438012:0:261255:0:(lov_object.c:1410:lov_object_init()) Process entered 00020000:00000001:0.0:1788572469.438015:0:261255:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.438020:0:261255:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626060. 00020000:00000010:0.0:1788572469.438023:0:261255:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106f00. 00020000:00000010:0.0:1788572469.438028:0:261255:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626ae0. 00020000:00000001:0.0:1788572469.438029:0:261255:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637856 : -109714647913760 : ffff9c3711626ae0) 00020000:00000002:0.0:1788572469.438033:0:261255:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626ae0, objid 0xdf8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.438037:0:261255:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.438042:0:261255:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106f00: ostid: 0x280000401:21133 ost idx: 0 gen: 0 00020000:00000001:0.0:1788572469.438044:0:261255:0:(lov_object.c:598:lov_init_composite()) Process entered 00020000:00000002:0.0:1788572469.438046:0:261255:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626ae0, objid 0xdf8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 2, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.438050:0:261255:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.438053:0:261255:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106f00: ostid: 0x280000401:21133 ost idx: 0 gen: 0 00020000:00000010:0.0:1788572469.438056:0:261255:0:(lov_object.c:618:lov_init_composite()) kmalloced '(comp->lo_mirrors)': 12 at ffff9c3731c905b0. 00020000:00000010:0.0:1788572469.438060:0:261255:0:(lov_object.c:622:lov_init_composite()) kmalloced '(comp->lo_entries)': 184 at ffff9c371eeb5480. 00020000:00000001:0.0:1788572469.438204:0:261255:0:(lov_object.c:189:lov_init_raid0()) Process entered 00020000:00000010:0.0:1788572469.438209:0:261255:0:(lov_object.c:194:lov_init_raid0()) kmalloced '(r0->lo_sub)': 8 at ffff9c371eb74990. 00020000:00000001:0.0:1788572469.438212:0:261255:0:(lov_object.c:83:lov_sub_find()) Process entered 00000020:00000001:0.0:1788572469.438214:0:261255:0:(lu_object.c:830:lu_object_find_at()) Process entered 00020000:00000001:0.0:1788572469.438219:0:261255:0:(lovsub_object.c:176:lovsub_object_alloc()) Process entered 00020000:00000010:0.0:1788572469.438296:0:261255:0:(lovsub_object.c:177:lovsub_object_alloc()) slab-alloced 'los': 200 at ffff9c3732214190. 00000020:00000001:0.0:1788572469.438299:0:261255:0:(cl_object.c:60:cl_object_header_init()) Process entered 00000020:00000001:0.0:1788572469.438301:0:261255:0:(cl_object.c:68:cl_object_header_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.438303:0:261255:0:(lovsub_object.c:190:lovsub_object_alloc()) Process leaving (rc=18446634359611015696 : -109714098535920 : ffff9c3732214210) 00020000:00000001:0.0:1788572469.438307:0:261255:0:(lovsub_object.c:32:lovsub_object_init()) Process entered 00000008:00000010:0.0:1788572469.438312:0:261255:0:(osc_object.c:435:osc_object_alloc()) slab-alloced 'osc': 448 at ffff9c3713d8b6c0. 00020000:00000001:0.0:1788572469.438314:0:261255:0:(lovsub_object.c:41:lovsub_object_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.438319:0:261255:0:(lu_object.c:929:lu_object_find_at()) Process leaving (rc=18446634359611015696 : -109714098535920 : ffff9c3732214210) 00020000:00000001:0.0:1788572469.438322:0:261255:0:(lov_object.c:87:lov_sub_find()) Process leaving (rc=18446634359611015696 : -109714098535920 : ffff9c3732214210) 00020000:00000002:0.0:1788572469.438326:0:261255:0:(lov_object.c:134:lov_init_sub()) [0x280000401:0x528d:0x0]@ffff9c3732214190[0:0] -> [0x200002b17:0xdf8:0x0]@ffff9c3732214af0: ostid: 0x280000401:21133 ost idx: 0 gen: 0 00020000:00000001:0.0:1788572469.438331:0:261255:0:(lov_object.c:259:lov_init_raid0()) Process leaving (rc=128 : 128 : 80) 00020000:00000001:0.0:1788572469.438334:0:261255:0:(lov_object.c:815:lov_init_composite()) Process leaving 00020000:00000001:0.0:1788572469.438336:0:261255:0:(lov_object.c:1438:lov_object_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.438339:0:261255:0:(lu_object.c:929:lu_object_find_at()) Process leaving (rc=18446634359611018096 : -109714098533520 : ffff9c3732214b70) 00000020:00001000:0.0:1788572469.438343:0:261255:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d120 00000080:00000001:0.0:1788572469.438347:0:261255:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329144 : 144115373145329144 : 200002b17000df8) 00000080:00000001:0.0:1788572469.438350:0:261255:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00000002:0.0:1788572469.438352:0:261255:0:(llite_lib.c:3050:ll_update_inode()) setting ino 144115373145329144 mtime from 0 to 1788572469 00000080:00000001:0.0:1788572469.438357:0:261255:0:(llite_lib.c:3378:ll_read_inode2()) Process leaving 00000080:00000001:0.0:1788572469.438359:0:261255:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911562824 : -109714797988792 : ffff9c3708707448) 00010000:00000001:0.0:1788572469.438361:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.438363:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.438364:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.438366:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.438368:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00000080:00000001:0.0:1788572469.438371:0:261255:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572469.438372:0:261255:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.438374:0:261255:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792808 : -109714643758808 : ffff9c3711a1d128) 00000020:00001000:0.0:1788572469.438375:0:261255:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d120 00000020:00000001:0.0:1788572469.438377:0:261255:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.438379:0:261255:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.438381:0:261255:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.438383:0:261255:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626e40. 00020000:00000010:0.0:1788572469.438386:0:261255:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.438389:0:261255:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116265a0. 00020000:00000001:0.0:1788572469.438391:0:261255:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636512 : -109714647915104 : ffff9c37116265a0) 00020000:00000002:0.0:1788572469.438393:0:261255:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116265a0, objid 0xdf8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.438397:0:261255:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.438399:0:261255:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106680: ostid: 0x280000401:21133 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572469.438402:0:261255:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3730ca36c8) owner ffff9c370259cf80 00020000:00000001:0.0:1788572469.438405:0:261255:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572469.438407:0:261255:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3730ca36c8) owner ffff9c370259cf80 00020000:00000010:0.0:1788572469.438411:0:261255:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.438415:0:261255:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626e40. 00020000:00000010:0.0:1788572469.438487:0:261255:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116265a0. 00020000:00000002:0.0:1788572469.438491:0:261255:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdf8:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572469.438493:0:261255:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.438495:0:261255:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.438498:0:261255:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572469.438500:0:261255:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca36c8) owner 0000000000000000/ffff9c370259cf80 00020000:00000002:0.0:1788572469.438503:0:261255:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3711626ae0 addref 2/0 by ffff9c370259cf80. 00020000:00000002:0.0:1788572469.438505:0:261255:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca36c8) owner 0000000000000000/ffff9c370259cf80 00020000:00000001:0.0:1788572469.438508:0:261255:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572469.438510:0:261255:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572469.438511:0:261255:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572469.438513:0:261255:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572469.438516:0:261255:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572469.438518:0:261255:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdf8:0x0]: layout version change: 4294967294 -> 0 00000020:00001000:0.0:1788572469.438521:0:261255:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d120 00000080:00000001:0.0:1788572469.438523:0:261255:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.438524:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.438525:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.438527:0:261255:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572469.438530:0:261255:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572469.438533:0:261255:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.438535:0:261255:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b17:0xdf8:0x0](ffff9c3708707448) 00000080:00010000:0.0:1788572469.438538:0:261255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdf8:0x0](ffff9c3708707448) for lock 0x1441586093168238 00800000:00000001:0.0:1788572469.438541:0:261255:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.438543:0:261255:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.438544:0:261255:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.438545:0:261255:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.438547:0:261255:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=3 00000020:00000001:0.0:1788572469.438549:0:261255:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.438778:0:261255:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.438782:0:261255:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.438783:0:261255:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.438784:0:261255:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.438786:0:261255:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.438788:0:261255:0:(namei.c:1596:ll_create_node()) Process leaving 00000100:00000001:0.0:1788572469.438791:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.438799:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 3 req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/664 e 0 to 0 dl 1788572554 ref 4 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.438807:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000002:0.0:1788572469.438811:0:261255:0:(namei.c:1651:ll_create_it()) inode ffff9c3708707448 need_sync_to_mds [0x200002b17:0xdf8:0x0] 00000080:00000001:0.0:1788572469.438815:0:261255:0:(namei.c:1686:ll_create_it()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.438819:0:261255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdf8:0x0](ffff9c3708707448) 00000080:00000001:0.0:1788572469.438822:0:261255:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572469.438826:0:261255:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000100:00000001:0.0:1788572469.438832:0:261255:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.438835:0:261255:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b5500 x1875440809927936/t25769859794(25769859794) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/664 e 0 to 0 dl 1788572554 ref 3 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.438841:0:261255:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.438845:0:261255:0:(pcc.c:3603:pcc_inode_create_fini()) Process entered 00000080:00000001:0.0:1788572469.438846:0:261255:0:(pcc.c:3606:pcc_inode_create_fini()) Process leaving (rc=0 : 0 : 0) 00000080:00000002:0.0:1788572469.438848:0:261255:0:(namei.c:1500:ll_atomic_open()) inode ffff9c3708707448 need_sync_to_mds [0x200002b17:0xdf8:0x0] 00000080:00000001:0.0:1788572469.438857:0:261255:0:(file.c:1086:ll_file_open()) Process entered 00000020:00000001:0.0:1788572469.438860:0:261255:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1603a68 00000080:00200008:0.0:1788572469.438862:0:261255:0:(file.c:1090:ll_file_open()) START file f434.sanity-100:[0x200002b17:0xdf8:0x0](ffff9c3708707448), flags 104501 00000800:00000200:0.0:1788572469.441472:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.441477:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572469.441482:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.441487:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.441492:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572469.441494:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572469.441497:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.441502:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4000 x1875440809928192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.441510:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.441513:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4000 x1875440809928192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.441519:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.441520:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.449957:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.449962:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5a00. 00000400:00000200:0.0:1788572469.449967:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.449972:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.449976:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c15d8 00000400:00000010:0.0:1788572469.449978:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c15d8. 00000100:00000001:0.0:1788572469.449981:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.449986:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71880 x1875440809928320/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.449994:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.449997:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71880 x1875440809928320/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.450003:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.450005:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.456370:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.456375:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572469.456379:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.456384:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.456388:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf13b8 00000400:00000010:0.0:1788572469.456390:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf13b8. 00000100:00000001:0.0:1788572469.456394:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.456398:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71880 x1875440809928448/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.456407:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.456410:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71880 x1875440809928448/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.456417:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.456418:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.464081:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.464090:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5a00. 00000400:00000200:0.0:1788572469.464100:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.464109:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.464119:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c17f8 00000400:00000010:0.0:1788572469.464122:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c17f8. 00000100:00000001:0.0:1788572469.464127:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.464135:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71180 x1875440809928704/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.464146:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.464150:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71180 x1875440809928704/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572485 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.464157:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.464159:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.705210:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.705219:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572469.705227:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.705235:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.705244:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1220 00000400:00000010:0.0:1788572469.705247:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1220. 00000100:00000001:0.0:1788572469.705253:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.705260:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809928832/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.705272:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.705276:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68380 x1875440809928832/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.705288:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.705291:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000002:00000010:0.0:1788572469.717780:0:261270:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d0913c0. 00010000:00000001:0.0:1788572469.717785:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.717786:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.717788:0:261270:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.717790:0:261270:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.717793:0:261270:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572469.717795:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.717797:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.717798:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.717800:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.717803:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00000002:00010000:0.0:1788572469.717806:0:261270:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.717813:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.717814:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.717817:0:261270:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572469.717818:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.717819:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.717821:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.717823:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.717824:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000040:0.0:1788572469.717828:0:261270:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25b00 count: 3 00010000:00000001:0.0:1788572469.717830:0:261270:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572469.717833:0:261270:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3477 (0 0) 00010000:00000001:0.0:1788572469.717836:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.717837:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.717839:0:261270:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572469.717842:0:261270:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c3f710 00000002:00002000:0.0:1788572469.717844:0:261270:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-001 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572469.717847:0:261270:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.717852:0:261270:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.717855:0:261270:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572469.717859:0:261270:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572469.717862:0:261270:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.717867:0:261270:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.717885:0:261270:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572469.717887:0:261270:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1c3fb20 it_disposition b 00000080:00000001:0.0:1788572469.717891:0:261270:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572469.717897:0:261270:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572469.717900:0:261270:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572469.717903:0:261270:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329045 : 144115373145329045 : 200002b17000d95) 00000080:00000001:0.0:1788572469.717906:0:261270:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572469.717914:0:261270:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.717919:0:261270:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792808 : -109714643758808 : ffff9c3711a1d128) 00000020:00001000:0.0:1788572469.717922:0:261270:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d120 00000020:00000001:0.0:1788572469.717925:0:261270:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.717931:0:261270:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.717933:0:261270:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.717940:0:261270:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116265a0. 00020000:00000010:0.0:1788572469.717944:0:261270:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.717951:0:261270:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626e40. 00020000:00000001:0.0:1788572469.717954:0:261270:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061638720 : -109714647912896 : ffff9c3711626e40) 00020000:00000002:0.0:1788572469.717958:0:261270:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626e40, objid 0xd95:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.717963:0:261270:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.717967:0:261270:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106680: ostid: 0x2c0000401:20969 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572469.717971:0:261270:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711ace7c0) owner ffff9c3718e2a7c0 00020000:00000001:0.0:1788572469.717975:0:261270:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572469.717977:0:261270:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711ace7c0) owner ffff9c3718e2a7c0 00020000:00000010:0.0:1788572469.717983:0:261270:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.717986:0:261270:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116265a0. 00020000:00000010:0.0:1788572469.717989:0:261270:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626e40. 00020000:00000002:0.0:1788572469.717993:0:261270:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xd95:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572469.717995:0:261270:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.717997:0:261270:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572469.718001:0:261270:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d120 00000080:00000001:0.0:1788572469.718017:0:261270:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329045 : 144115373145329045 : 200002b17000d95) 00000080:00000001:0.0:1788572469.718020:0:261270:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572469.718027:0:261270:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xd95:0x0](ffff9c372f0fcb08): rc = 0 00000080:00000001:0.0:1788572469.718029:0:261270:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359559539464 : -109714150012152 : ffff9c372f0fcb08) 00010000:00000001:0.0:1788572469.718031:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.718033:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.718034:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.718036:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718038:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00000080:00000001:0.0:1788572469.718041:0:261270:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572469.718042:0:261270:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.718044:0:261270:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792808 : -109714643758808 : ffff9c3711a1d128) 00000020:00001000:0.0:1788572469.718046:0:261270:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d120 00000020:00000001:0.0:1788572469.718047:0:261270:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.718048:0:261270:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.718049:0:261270:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.718052:0:261270:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626e40. 00020000:00000010:0.0:1788572469.718054:0:261270:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.718056:0:261270:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116265a0. 00020000:00000001:0.0:1788572469.718058:0:261270:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636512 : -109714647915104 : ffff9c37116265a0) 00020000:00000002:0.0:1788572469.718060:0:261270:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116265a0, objid 0xd95:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572469.718063:0:261270:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572469.718067:0:261270:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106680: ostid: 0x2c0000401:20969 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572469.718069:0:261270:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711ace7c0) owner ffff9c3718e2a7c0 00020000:00000001:0.0:1788572469.718071:0:261270:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572469.718073:0:261270:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711ace7c0) owner ffff9c3718e2a7c0 00020000:00000010:0.0:1788572469.718075:0:261270:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106680. 00020000:00000010:0.0:1788572469.718078:0:261270:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626e40. 00020000:00000010:0.0:1788572469.718081:0:261270:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116265a0. 00020000:00000002:0.0:1788572469.718083:0:261270:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xd95:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572469.718084:0:261270:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718086:0:261270:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718090:0:261270:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572469.718093:0:261270:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711ace7c0) owner 0000000000000000/ffff9c3718e2a7c0 00020000:00000002:0.0:1788572469.718096:0:261270:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3711bdd360 addref 2/0 by ffff9c3718e2a7c0. 00020000:00000002:0.0:1788572469.718099:0:261270:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711ace7c0) owner 0000000000000000/ffff9c3718e2a7c0 00020000:00000001:0.0:1788572469.718101:0:261270:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572469.718104:0:261270:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572469.718105:0:261270:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572469.718107:0:261270:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572469.718109:0:261270:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572469.718112:0:261270:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xd95:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572469.718114:0:261270:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d120 00000080:00000001:0.0:1788572469.718116:0:261270:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718118:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718119:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.718121:0:261270:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572469.718123:0:261270:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572469.718125:0:261270:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.718129:0:261270:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xd95:0x0](ffff9c372f0fcb08) for lock 0x1441586093168246 00800000:00000001:0.0:1788572469.718132:0:261270:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.718133:0:261270:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.718135:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.718136:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.718137:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.718139:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718141:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718143:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718144:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.718145:0:261270:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.718147:0:261270:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572469.718154:0:261270:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c3702109bd0 inode ffff9c372f0fcb08 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572469.718157:0:261270:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572469.718161:0:261270:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xd95:0x0](ffff9c372f0fcb08) 00000080:00010000:0.0:1788572469.718166:0:261270:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168246 from it ffffb95bc1c3fb20 00010000:00000001:0.0:1788572469.718168:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.718169:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.718170:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.718172:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718174:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718176:0:261270:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.718178:0:261270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.718184:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718186:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.718187:0:261270:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.718189:0:261270:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.718191:0:261270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.718198:0:261270:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.718200:0:261270:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.718201:0:261270:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.718206:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718207:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.718208:0:261270:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718211:0:261270:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.718213:0:261270:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718215:0:261270:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718218:0:261270:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718219:0:261270:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.718220:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718221:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.718223:0:261270:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572469.718227:0:261270:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c21e00. 00000100:00000001:0.0:1788572469.718233:0:261270:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.718238:0:261270:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440809928832/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572554 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.718244:0:261270:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572469.718247:0:261270:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572469.718252:0:261270:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37051a7000. 02000000:00000001:0.0:1788572469.718255:0:261270:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572469.718257:0:261270:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572469.718259:0:261270:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572469.718262:0:261270:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572469.718265:0:261270:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788572469.718268:0:261270:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572469.718270:0:261270:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572469.718272:0:261270:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788572469.718275:0:261270:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572469.718277:0:261270:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.718280:0:261270:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572469.718281:0:261270:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1c3fb20 released 00000080:00000001:0.0:1788572469.718283:0:261270:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572469.718297:0:261270:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c3fd08 00000080:00200008:0.0:1788572469.718300:0:261270:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-001:[0x200002b17:0xd95:0x0](ffff9c372f0fcb08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572469.718306:0:261270:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572469.718307:0:261270:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c3fc80 00000080:00200000:0.0:1788572469.718309:0:261270:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xd95:0x0](ffff9c372f0fcb08),name=f434.sanity-001 00000080:00000010:0.0:1788572469.718316:0:261270:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572469.718322:0:261270:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.718329:0:261270:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572469.718332:0:261270:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.718334:0:261270:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c3fba0 00800000:00000002:0.0:1788572469.718336:0:261270:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xd95:0x0] 'f434.sanity-001' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.718339:0:261270:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572469.718343:0:261270:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.718346:0:261270:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.718348:0:261270:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.718351:0:261270:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.718353:0:261270:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.718355:0:261270:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.718357:0:261270:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.718360:0:261270:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.718361:0:261270:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.718362:0:261270:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.718363:0:261270:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.718365:0:261270:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.718367:0:261270:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd95:0x0] 00800000:00000001:0.0:1788572469.718368:0:261270:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.718371:0:261270:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xd95:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572469.718374:0:261270:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.718376:0:261270:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c3fa70 00000002:00010000:0.0:1788572469.718378:0:261270:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xd95:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572469.718381:0:261270:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572469.718383:0:261270:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.718384:0:261270:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.718390:0:261270:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25b00 count: 3 00010000:00000001:0.0:1788572469.718392:0:261270:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.718394:0:261270:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.718396:0:261270:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572469.718399:0:261270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572469.718407:0:261270:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.718411:0:261270:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.718414:0:261270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572469.718419:0:261270:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572469.718421:0:261270:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572469.718423:0:261270:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.718424:0:261270:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.718426:0:261270:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572469.718439:0:261270:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572469.718442:0:261270:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572469.718444:0:261270:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572469.718446:0:261270:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572469.718452:0:261270:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572469.718454:0:261270:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.718455:0:261270:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xd95:0x0](ffff9c372f0fcb08) 00000080:00010000:0.0:1788572469.718459:0:261270:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xd95:0x0](ffff9c372f0fcb08) for lock 0x1441586093168246 00800000:00000001:0.0:1788572469.718461:0:261270:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.718463:0:261270:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.718464:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.718466:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.718467:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.718469:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718471:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718473:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718474:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.718475:0:261270:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.718476:0:261270:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.718478:0:261270:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168246 from it ffffb95bc1c3fcb8 00010000:00000001:0.0:1788572469.718480:0:261270:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.718481:0:261270:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.718482:0:261270:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119a00 refcount=3 00000020:00000001:0.0:1788572469.718483:0:261270:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718485:0:261270:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291222528 : -109714418329088 : ffff9c371f119a00) 00010000:00000001:0.0:1788572469.718487:0:261270:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.718489:0:261270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.718495:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718496:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.718497:0:261270:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.718499:0:261270:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.718501:0:261270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119a00/0x1441586093168246 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd95:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1bd0 expref: -99 pid: 261270 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572469.718507:0:261270:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.718508:0:261270:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.718509:0:261270:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.718511:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718513:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.718514:0:261270:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718516:0:261270:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.718518:0:261270:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718519:0:261270:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718521:0:261270:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718523:0:261270:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.718524:0:261270:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.718525:0:261270:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572469.718527:0:261270:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572469.718540:0:261270:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.718543:0:261270:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572469.718546:0:261270:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572469.718548:0:261270:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.718550:0:261270:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572469.718551:0:261270:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.718553:0:261270:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572469.718556:0:261270:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572469.718558:0:261270:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.718560:0:261270:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065792808 : -109714643758808 : ffff9c3711a1d128) 00000020:00001000:0.0:1788572469.718562:0:261270:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d120 00000020:00000001:0.0:1788572469.718567:0:261270:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1850 -> ffff9c37313e1850 00000020:00000001:0.0:1788572469.718569:0:261270:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572469.718572:0:261270:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572469.718574:0:261270:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd95:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572469.718577:0:261270:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.718578:0:261270:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572469.718580:0:261270:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572469.718581:0:261270:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.718583:0:261270:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572469.718586:0:261270:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd95:0x0]io ffff9c37051799d0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572469.718589:0:261270:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711ace7c0) owner 0000000000000000/ffff9c3718e2a7c0 00020000:00000001:0.0:1788572469.718592:0:261270:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572469.718594:0:261270:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572469.718596:0:261270:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572469.718598:0:261270:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718600:0:261270:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572469.718603:0:261270:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572469.718604:0:261270:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718612:0:261270:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.718613:0:261270:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572469.718614:0:261270:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572469.718616:0:261270:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711ace7c0) owner 0000000000000000/ffff9c3718e2a7c0 00000020:00000001:0.0:1788572469.718618:0:261270:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572469.718620:0:261270:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572469.718622:0:261270:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xd95:0x0] 00000020:00000001:0.0:1788572469.718624:0:261270:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572469.718626:0:261270:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572469.718628:0:261270:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572469.718630:0:261270:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572469.718663:0:261270:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572469.718670:0:261270:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676400. 00020000:00000002:0.0:1788572469.718672:0:261270:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572469.718674:0:261270:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572469.718677:0:261270:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572469.718679:0:261270:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572469.718681:0:261270:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.718682:0:261270:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795976 : -109714643755640 : ffff9c3711a1dd88) 00000020:00001000:0.0:1788572469.718684:0:261270:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dd80 00000020:00000001:0.0:1788572469.718687:0:261270:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1788 -> ffff9c37313e1850 00000020:00000001:0.0:1788572469.718688:0:261270:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572469.718691:0:261270:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572469.718692:0:261270:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572469.718694:0:261270:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718695:0:261270:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718697:0:261270:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859899552 : -109714849652064 : ffff9c37055c22a0) 00000020:00000001:0.0:1788572469.718699:0:261270:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572469.718704:0:261270:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a71d0. 00000008:00010000:0.0:1788572469.718706:0:261270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572469.718709:0:261270:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572469.718710:0:261270:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572469.718711:0:261270:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676430, osc lock ffff9c37030a71d0, flags 0x400001002 00000020:00000001:0.0:1788572469.718713:0:261270:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718715:0:261270:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718717:0:261270:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725824 : -109714177825792 : ffff9c372d676400) 00000020:00000001:0.0:1788572469.718719:0:261270:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572469.718720:0:261270:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572469.718722:0:261270:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718724:0:261270:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718726:0:261270:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572469.718728:0:261270:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572469.718730:0:261270:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572469.718731:0:261270:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572469.718732:0:261270:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572469.718733:0:261270:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859899552 : -109714849652064 : ffff9c37055c22a0) 00000020:00000001:0.0:1788572469.718735:0:261270:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572469.718737:0:261270:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572469.718739:0:261270:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572469.718742:0:261270:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572469.718744:0:261270:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.718746:0:261270:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572469.718747:0:261270:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572469.718748:0:261270:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572469.718750:0:261270:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572469.718752:0:261270:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572469.718754:0:261270:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572469.718758:0:261270:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718760:0:261270:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572469.718764:0:261270:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572469.718767:0:261270:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572469.718771:0:261270:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572469.718776:0:261270:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.718778:0:261270:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.718780:0:261270:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.718783:0:261270:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.718785:0:261270:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.718791:0:261270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7e00. 00000100:00000001:0.0:1788572469.718798:0:261270:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718801:0:261270:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.718803:0:261270:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718804:0:261270:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.718805:0:261270:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718807:0:261270:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.718810:0:261270:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00010000:00000001:0.0:1788572469.718812:0:261270:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572469.718815:0:261270:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572469.718819:0:261270:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76e00. 00010000:00000010:0.0:1788572469.718823:0:261270:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3708c76f00. 00010000:00000001:0.0:1788572469.718828:0:261270:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572469.718832:0:261270:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c371fd1a200. 00000020:00000001:0.0:1788572469.718837:0:261270:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572469.718839:0:261270:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c371fd1a200 with handle 0x144158609316824d to hash 00000020:00000001:0.0:1788572469.718841:0:261270:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572469.718842:0:261270:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000010:0.0:1788572469.718846:0:261270:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d0915c0. 00010000:00000001:0.0:1788572469.718848:0:261270:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.718850:0:261270:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.718852:0:261270:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.718855:0:261270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.718866:0:261270:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.718887:0:261270:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.718892:0:261270:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.718896:0:261270:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.718904:0:261270:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572469.718940:0:261270:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572469.718943:0:261270:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718945:0:261270:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572469.718947:0:261270:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718949:0:261270:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.718951:0:261270:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572469.718953:0:261270:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572469.718955:0:261270:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572469.718957:0:261270:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572469.718968:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572469.718972:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572469.718973:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.718980:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.718983:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.718985:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.718987:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.718989:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.718995:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.719002:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.719004:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.719007:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.719009:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.719012:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.719015:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809928960:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.719019:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.719024:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.719027:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.719029:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.719034:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572469.719036:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.719043:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.719047:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.719055:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572469.719061:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809928960, portal 4 00000100:00000001:0.0:1788572469.719064:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.719065:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572469.719071:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.719077:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.719080:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.719082:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572469.719084:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809928960, offset 0 00000400:00000200:0.0:1788572469.719091:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.719099:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.719105:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.719111:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.719115:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.719118:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.719121:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.719125:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539637:539637:256:4294967295] 192.168.202.148@tcp LPNI seq info [539602:539602:8:4294967295] 00000400:00000200:0.0:1788572469.719129:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.719138:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.719144:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.719149:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572469.719156:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.719163:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.719168:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.719179:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.719183:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 183641 usecs (184 jiffies) 00000400:00000200:0.0:1788572469.719186:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.719188:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307608909 00000100:00000001:0.0:1788572469.719191:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.719193:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307608909 00000100:00000001:0.0:1788572469.719195:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.719197:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.719200:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.719203:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.719205:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572469.719303:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.719307:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572469.719312:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.719317:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.719323:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31a18 00000400:00000010:0.0:1788572469.719325:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31a18. 00000100:00000001:0.0:1788572469.719329:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.719333:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.719341:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.719344:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.719350:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.719352:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572469.724479:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.724485:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.724493:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.724495:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.724501:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572469.724506:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed315d8 00000400:00000010:0.0:1788572469.724509:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed315d8. 00000100:00000001:0.0:1788572469.724513:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.724518:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.725884:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.725908:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.725912:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.725915:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.725919:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.725921:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.725923:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.725926:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.725938:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=33 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572469.725943:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=6 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572469.725946:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.725947:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.725950:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.725954:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.725958:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.725959:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572469.725961:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.725964:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.725971:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.725980:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.725983:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.725985:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572469.725987:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572469.725990:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.725992:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.725996:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=4 00000020:00000001:0.0:1788572469.725998:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726001:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726013:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.726014:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.726016:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=5 00000020:00000001:0.0:1788572469.726017:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726019:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726022:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.726025:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.726029:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726041:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726043:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.726045:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572469.726047:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.726048:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.726049:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=6 00000020:00000001:0.0:1788572469.726051:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726053:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000040:0.0:1788572469.726060:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c371fd1a200, remote cookie: 0xd9894672d99a1bd7, flags: 0x1001 00010000:00010000:0.0:1788572469.726065:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726073:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572469.726076:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726078:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572469.726082:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572469.726086:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572469.726094:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51e9:0x0].0x0 (ffff9c3708c76e00) refcount = 1 00010000:00000040:0.0:1788572469.726096:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572469.726099:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726108:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572469.726110:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.726113:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572469.726115:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.726118:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726125:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572469.726126:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726127:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.726128:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726129:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.726131:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572469.726134:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572469.726137:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.726138:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.726139:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=5 00000020:00000001:0.0:1788572469.726141:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726143:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00000008:00000001:0.0:1788572469.726147:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572469.726151:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8e00 -> ffff9c37313e1850 00000008:00010000:0.0:1788572469.726155:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572469.726163:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8e00 -> ffff9c37313e1850 00000020:00000001:0.0:1788572469.726165:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572469.726167:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572469.726170:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572469.726173:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.726175:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.726177:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8e00 -> ffff9c37313e1850 00000008:00000001:0.0:1788572469.726179:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572469.726182:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572469.726183:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572469.726200:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572469.726202:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726204:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.726205:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.726207:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=6 00000020:00000001:0.0:1788572469.726209:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726211:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726214:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.726217:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726224:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726225:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572469.726229:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726236:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.726238:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726239:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.726240:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726242:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.726243:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.726244:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1a200 refcount=5 00000020:00000001:0.0:1788572469.726246:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726248:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303807488 : -109714405744128 : ffff9c371fd1a200) 00010000:00000001:0.0:1788572469.726250:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.726252:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.726259:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726260:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.726261:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.726263:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.726266:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000000ee0f3ac/0x144158609316824d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a1bd7 expref: -99 pid: 261270 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.726274:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.726276:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.726278:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.726282:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726283:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.726284:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726288:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.726290:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726293:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726296:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.726297:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.726299:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726301:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.726302:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.726303:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.726305:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.726311:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.726321:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809928960:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.726327:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.726331:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.726335:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440809928960/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.726342:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572469.726346:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572469.726353:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572469.726362:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572469.726365:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572469.726372:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572469.726375:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572469.726381:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7e00. 02000000:00000001:0.0:1788572469.726391:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572469.726395:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572469.726399:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788572469.726404:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572469.726409:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.726415:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.726425:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.726431:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.726443:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.726449:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.759228:0:261272:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572469.759236:0:261272:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572469.759239:0:261272:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572469.759242:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57b20 00000080:00200000:0.0:1788572469.759247:0:261272:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572469.759257:0:261272:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572469.759265:0:261272:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572469.759269:0:261272:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572469.759274:0:261272:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.759277:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57a40 00800000:00000002:0.0:1788572469.759280:0:261272:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572469.759283:0:261272:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572469.759286:0:261272:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.759289:0:261272:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.759291:0:261272:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.759293:0:261272:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.759295:0:261272:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.759297:0:261272:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572469.759299:0:261272:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.759302:0:261272:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572469.759307:0:261272:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.759309:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57910 00000002:00010000:0.0:1788572469.759311:0:261272:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572469.759314:0:261272:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572469.759316:0:261272:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.759320:0:261272:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.759328:0:261272:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572469.759331:0:261272:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.759337:0:261272:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.759341:0:261272:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572469.759346:0:261272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 02000000:00000001:0.0:1788572469.759358:0:261272:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.759363:0:261272:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.759366:0:261272:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 00000002:00000001:0.0:1788572469.759373:0:261272:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572469.759376:0:261272:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572469.759378:0:261272:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.759380:0:261272:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572469.759382:0:261272:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572469.759384:0:261272:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572469.759387:0:261272:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572469.759390:0:261272:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572469.759393:0:261272:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572469.759404:0:261272:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572469.759406:0:261272:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.759408:0:261272:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572469.759412:0:261272:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572469.759415:0:261272:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572469.759417:0:261272:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572469.759419:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.759421:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.759424:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572469.759426:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.759603:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.759606:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.759608:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.759609:0:261272:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.759611:0:261272:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572469.759614:0:261272:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c57b58 00010000:00000001:0.0:1788572469.759617:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.759618:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.759619:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572469.759621:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.759623:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572469.759626:0:261272:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.759629:0:261272:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 00010000:00000001:0.0:1788572469.759653:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.759655:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.759656:0:261272:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.759659:0:261272:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.759662:0:261272:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 103 lru_type: 1 00010000:00000001:0.0:1788572469.759671:0:261272:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.759674:0:261272:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.759676:0:261272:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.759681:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.759682:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.759683:0:261272:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759686:0:261272:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.759688:0:261272:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759690:0:261272:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759693:0:261272:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759695:0:261272:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.759696:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.759698:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572469.759701:0:261272:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572469.759702:0:261272:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572469.759704:0:261272:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572469.759709:0:261272:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.759713:0:261272:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572469.759715:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57ae8 00000080:00002000:0.0:1788572469.759717:0:261272:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 1 00000080:00000001:0.0:1788572469.759719:0:261272:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.759721:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57b30 00000080:00200000:0.0:1788572469.759722:0:261272:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572469.759727:0:261272:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572469.759731:0:261272:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572469.759733:0:261272:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572469.759735:0:261272:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572469.759745:0:261272:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fe80. 00000020:00000001:0.0:1788572469.759748:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57ae8 00000080:00200000:0.0:1788572469.759750:0:261272:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-002, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572469.759753:0:261272:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572469.759755:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57998 00000080:00200000:0.0:1788572469.759757:0:261272:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-002, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572469.759763:0:261272:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572469.759766:0:261272:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.759769:0:261272:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572469.759771:0:261272:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.759772:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c578b8 00800000:00000002:0.0:1788572469.759774:0:261272:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-002' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.759777:0:261272:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572469.759780:0:261272:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.759782:0:261272:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.759783:0:261272:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.759785:0:261272:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.759787:0:261272:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.759789:0:261272:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.759791:0:261272:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.759793:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57850 00800000:00000002:0.0:1788572469.759795:0:261272:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-002' -> mds #0 00000002:00000001:0.0:1788572469.759797:0:261272:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572469.759799:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57788 00000002:00010000:0.0:1788572469.759800:0:261272:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-002,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572469.759804:0:261272:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572469.759807:0:261272:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572469.759812:0:261272:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572469.759816:0:261272:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572469.759822:0:261272:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572469.759825:0:261272:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.759827:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.759828:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.759829:0:261272:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.759832:0:261272:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.759834:0:261272:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.759837:0:261272:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.759843:0:261272:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572469.759850:0:261272:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759854:0:261272:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.759856:0:261272:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759858:0:261272:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.759859:0:261272:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.759861:0:261272:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.759867:0:261272:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00010000:00000001:0.0:1788572469.759880:0:261272:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572469.759882:0:261272:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572469.759886:0:261272:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572469.759888:0:261272:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.759891:0:261272:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572469.759895:0:261272:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c371fd1ac00. 00000020:00000001:0.0:1788572469.759898:0:261272:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572469.759900:0:261272:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c371fd1ac00 with handle 0x1441586093168254 to hash 00000020:00000001:0.0:1788572469.759902:0:261272:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572469.759904:0:261272:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.759907:0:261272:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.759909:0:261272:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.759911:0:261272:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.759915:0:261272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261272 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.759922:0:261272:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261272 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.759932:0:261272:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261272 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572469.759937:0:261272:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572469.759939:0:261272:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572469.759945:0:261272:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572469.759947:0:261272:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572469.759950:0:261272:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.759951:0:261272:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788572469.759956:0:261272:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.759960:0:261272:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.759962:0:261272:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572469.759964:0:261272:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.759974:0:261272:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.759983:0:261272:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.759985:0:261272:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.759988:0:261272:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.759990:0:261272:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.759993:0:261272:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.759996:0:261272:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261272:1875440809929088:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.760000:0:261272:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.760006:0:261272:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.760008:0:261272:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.760011:0:261272:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.760015:0:261272:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572469.760017:0:261272:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.760024:0:261272:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.760029:0:261272:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.760036:0:261272:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572469.760043:0:261272:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809929088, portal 10 00000100:00000001:0.0:1788572469.760045:0:261272:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.760046:0:261272:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572469.760051:0:261272:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.760058:0:261272:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.760060:0:261272:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.760063:0:261272:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572469.760065:0:261272:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809929088, offset 0 00000400:00000200:0.0:1788572469.760071:0:261272:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.760080:0:261272:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.760087:0:261272:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.760093:0:261272:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.760097:0:261272:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.760218:0:261272:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.760223:0:261272:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.760227:0:261272:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539638:539638:256:4294967295] 192.168.202.148@tcp LPNI seq info [539603:539603:8:4294967295] 00000400:00000200:0.0:1788572469.760232:0:261272:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.760241:0:261272:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.760248:0:261272:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.760253:0:261272:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572469.760259:0:261272:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.760266:0:261272:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.760270:0:261272:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.760295:0:261272:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.760299:0:261272:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14911 usecs (15 jiffies) 00000400:00000200:0.0:1788572469.760303:0:261272:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.760305:0:261272:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307608781 00000100:00000001:0.0:1788572469.760308:0:261272:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.760311:0:261272:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307608781 00000100:00000001:0.0:1788572469.760313:0:261272:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.760315:0:261272:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.760317:0:261272:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.760320:0:261272:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.760321:0:261272:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572469.760325:0:261272:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572469.760327:0:261272:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.760330:0:261272:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.771376:0:261272:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.771945:0:261272:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.771948:0:261272:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.771953:0:261272:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572469.771958:0:261272:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed315d8 00000400:00000010:0.0:1788572469.771961:0:261272:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed315d8. 00000100:00000001:0.0:1788572469.771966:0:261272:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.771972:0:261272:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.771981:0:261272:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.771987:0:261272:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.771989:0:261272:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.771992:0:261272:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.771996:0:261272:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.771998:0:261272:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.772001:0:261272:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.772004:0:261272:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.772014:0:261272:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=33 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572469.772018:0:261272:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=65 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572469.772021:0:261272:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.772023:0:261272:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.772026:0:261272:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.772029:0:261272:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.772033:0:261272:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.772036:0:261272:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572469.772038:0:261272:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.772041:0:261272:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.772046:0:261272:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.772053:0:261272:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.772056:0:261272:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.772057:0:261272:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572469.772060:0:261272:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572554 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.772142:0:261272:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261272:1875440809929088:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.772147:0:261272:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.772149:0:261272:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.772152:0:261272:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572469.772154:0:261272:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.772158:0:261272:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572554 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.772164:0:261272:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572469.772167:0:261272:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572469.772172:0:261272:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572469.772173:0:261272:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.772176:0:261272:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572469.772180:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772183:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772186:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=4 00000020:00000001:0.0:1788572469.772188:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772190:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000040:0.0:1788572469.772198:0:261272:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c371fd1ac00, remote cookie: 0xd9894672d99a1bf3, flags: 0x1001 00010000:00010000:0.0:1788572469.772203:0:261272:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572469.772211:0:261272:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xd96:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572469.772214:0:261272:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572469.772221:0:261272:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7500 count: 2 00010000:00000001:0.0:1788572469.772224:0:261272:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.772227:0:261272:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.772230:0:261272:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.772237:0:261272:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772243:0:261272:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572469.772247:0:261272:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572469.772249:0:261272:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572469.772252:0:261272:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572469.772253:0:261272:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572469.772255:0:261272:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xd96:0x0].0x0 (ffff9c37050f7500) refcount = 2 00010000:00000040:0.0:1788572469.772258:0:261272:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572469.772260:0:261272:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421ca400/0x1441586093167ce3 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xd96:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a08f2 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572469.772267:0:261272:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772273:0:261272:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572469.772276:0:261272:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572469.772278:0:261272:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.772281:0:261272:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572469.772283:0:261272:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.772286:0:261272:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772292:0:261272:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572469.772293:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772295:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.772297:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772298:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.772299:0:261272:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.772304:0:261272:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572469.772305:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772306:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772307:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=3 00000020:00000001:0.0:1788572469.772310:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772311:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772314:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772315:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772316:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=4 00000020:00000001:0.0:1788572469.772318:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772321:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772323:0:261272:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.772326:0:261272:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.772329:0:261272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772335:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772336:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.772337:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772338:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772339:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=5 00000020:00000001:0.0:1788572469.772341:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772343:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772345:0:261272:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.772348:0:261272:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772353:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772355:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572469.772358:0:261272:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772364:0:261272:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.772365:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772366:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.772367:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772368:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572469.772374:0:261272:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b5f80 x1875440809929088/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572554 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572469.772383:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772384:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772385:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=3 00000020:00000001:0.0:1788572469.772388:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772390:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00000002:00010000:0.0:1788572469.772393:0:261272:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572469.772405:0:261272:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091e80. 00010000:00000001:0.0:1788572469.772407:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772408:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.772410:0:261272:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.772412:0:261272:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.772414:0:261272:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572469.772417:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772418:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772419:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=3 00000020:00000001:0.0:1788572469.772421:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772423:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00000002:00010000:0.0:1788572469.772491:0:261272:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1ac00/0x1441586093168254 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd96:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bf3 expref: -99 pid: 261272 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.772500:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772501:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.772503:0:261272:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572469.772504:0:261272:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.772506:0:261272:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.772507:0:261272:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371fd1ac00 refcount=3 00000020:00000001:0.0:1788572469.772509:0:261272:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000001:0.0:1788572469.772511:0:261272:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359303810048 : -109714405741568 : ffff9c371fd1ac00) 00010000:00000040:0.0:1788572469.772514:0:261272:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7500 count: 3 00010000:00000001:0.0:1788572469.772516:0:261272:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572469.772520:0:261272:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3478 (0 0) 00010000:00000001:0.0:1788572469.772523:0:261272:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.772524:0:261272:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.772526:0:261272:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572469.772528:0:261272:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c57710 00000002:00002000:0.0:1788572469.772531:0:261272:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-002 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572469.772534:0:261272:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.772537:0:261272:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.772540:0:261272:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572469.772543:0:261272:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572469.772546:0:261272:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.772550:0:261272:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572469.772552:0:261272:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572469.772554:0:261272:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1c57b20 it_disposition b 00000080:00000001:0.0:1788572469.772558:0:261272:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572469.772565:0:261272:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572469.772568:0:261272:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572469.772570:0:261272:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329046 : 144115373145329046 : 200002b17000d96) 00000080:00000001:0.0:1788572469.772573:0:261272:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572469.772580:0:261272:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572469.772584:0:261272:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359600765480 : -109714108786136 : ffff9c373184da28) 00000020:00001000:0.0:1788572469.772586:0:261272:0:(cl_object.c:823:cl_env_get()) 1@ffff9c373184da20 00000020:00000001:0.0:1788572469.772588:0:261272:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572469.772593:0:261272:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572469.772596:0:261272:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572469.772601:0:261272:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116265a0. 00020000:00000010:0.0:1788572469.772925:0:261272:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106680. 00000100:00000001:0.0:1788572469.778786:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.778791:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.778793:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.778800:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.778809:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.778812:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.778826:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.778829:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.778831:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.778835:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809929216:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.778839:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.778846:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.778848:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.778851:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.778856:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572469.778858:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.778866:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.778879:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.778887:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572469.778894:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809929216, portal 4 00000100:00000001:0.0:1788572469.778897:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.778898:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788572469.778905:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.778912:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.778914:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.778917:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572469.778920:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809929216, offset 0 00000400:00000200:0.0:1788572469.778926:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.778935:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.778942:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.778948:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.778952:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.778956:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.778959:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.778962:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539639:539639:256:4294967295] 192.168.202.148@tcp LPNI seq info [539604:539604:8:4294967295] 00000400:00000200:0.0:1788572469.778966:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.778975:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.778981:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.778987:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572469.778993:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.779002:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.779005:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.779025:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.779029:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 173202 usecs (174 jiffies) 00000400:00000200:0.0:1788572469.779033:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.779036:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307608959 00000100:00000001:0.0:1788572469.779039:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.779041:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307608959 00000100:00000001:0.0:1788572469.779043:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.779045:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.779049:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.779052:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.779054:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.780782:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.780787:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572469.780792:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.780794:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.785234:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.785239:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572469.786060:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572469.786064:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572469.786068:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572469.786182:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed315d8 00000400:00000010:0.0:1788572469.786184:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed315d8. 00000100:00000001:0.0:1788572469.786189:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.786196:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572469.786206:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.786212:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572469.786214:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.786217:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572469.786221:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572469.786223:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572469.786225:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.786228:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572469.786239:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=33 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572469.786243:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=84 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572469.786245:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572469.786247:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786250:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572469.786253:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.786257:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572469.786259:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572469.786260:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572469.786263:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.786267:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.786275:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.786277:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572469.786279:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572469.786282:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572469.786285:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786288:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786291:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=4 00000020:00000001:0.0:1788572469.786293:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786296:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786298:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786300:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786301:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=5 00000020:00000001:0.0:1788572469.786303:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786304:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786308:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.786312:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.786316:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: --/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786327:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786330:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.786331:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572469.786334:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786335:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786336:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=6 00000020:00000001:0.0:1788572469.786338:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786339:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000040:0.0:1788572469.786346:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37421ca000, remote cookie: 0xd9894672d99a1bfa, flags: 0x1001 00010000:00010000:0.0:1788572469.786351:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: --/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786360:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572469.786363:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786365:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572469.786368:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572469.786373:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572469.786381:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x525c:0x0].0x0 (ffff9c37050f7300) refcount = 1 00010000:00000040:0.0:1788572469.786383:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572469.786386:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786394:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572469.786396:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.786399:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572469.786401:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.786404:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786411:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572469.786413:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786414:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.786415:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786416:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.786418:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572469.786420:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572469.786424:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786425:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786427:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=5 00000020:00000001:0.0:1788572469.786429:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786430:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00000008:00000001:0.0:1788572469.786434:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572469.786438:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000008:00010000:0.0:1788572469.786442:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572469.786450:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000020:00000001:0.0:1788572469.786451:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572469.786454:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572469.786456:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572469.786459:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.786461:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.786463:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000008:00000001:0.0:1788572469.786464:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572469.786467:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572469.786469:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572469.786483:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572469.786485:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786487:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786489:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786490:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=6 00000020:00000001:0.0:1788572469.786492:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786494:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786497:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.786500:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786508:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786509:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572469.786511:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786518:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.786519:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786520:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.786521:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786523:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.786524:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.786525:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421ca000 refcount=5 00000020:00000001:0.0:1788572469.786527:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786528:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879147520 : -109713830404096 : ffff9c37421ca000) 00010000:00000001:0.0:1788572469.786530:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572469.786533:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.786539:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786540:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.786541:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572469.786543:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572469.786546:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572469.786554:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572469.786556:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572469.786558:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.786562:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786563:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.786565:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786568:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572469.786570:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786572:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786574:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.786576:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572469.786577:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786578:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.786579:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.786580:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572469.786582:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.786587:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572469.786596:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809929216:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.786602:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.786605:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.786608:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440809929216/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.786615:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572469.786617:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572469.786621:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572469.786625:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572469.786627:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572469.786628:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572469.786631:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572469.786633:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215c00. 02000000:00000001:0.0:1788572469.788499:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572469.788502:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572469.788505:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788572469.788509:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572469.788511:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.788514:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.788524:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.788526:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.788528:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.788532:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572469.818207:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.818215:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572469.818224:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.818233:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.818240:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572469.818242:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572469.818247:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.818253:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809929344/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.818265:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.818269:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5f80 x1875440809929344/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.818275:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.818277:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.823364:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.823369:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572469.823373:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.823378:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.823383:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1000 00000400:00000010:0.0:1788572469.823385:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1000. 00000100:00000001:0.0:1788572469.823388:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.823392:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809929472/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.823400:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.823403:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440809929472/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.823408:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.823410:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.869371:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.869375:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572469.869380:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.869385:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.869391:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1440 00000400:00000010:0.0:1788572469.869393:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1440. 00000100:00000001:0.0:1788572469.869396:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.869400:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70a80 x1875440809929856/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3472/7976 e 0 to 0 dl 1788572554 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.869409:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.869412:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440809929856/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3472/7976 e 0 to 0 dl 1788572554 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.869419:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.869420:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572469.869432:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572469.869435:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572469.869437:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.869440:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572469.869441:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.869443:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572469.869444:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572469.869446:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572469.869450:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4e00 x1875440809929728/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.869457:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572469.869458:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.869460:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572469.869462:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572469.869465:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572469.869467:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809929728:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572469.869470:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572469.869473:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572469.869475:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572469.869477:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572469.869480:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788572469.869482:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572469.869486:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572469.869489:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572469.869493:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572469.869496:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809929728, portal 4 00000100:00000001:0.0:1788572469.869498:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572469.869499:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923968 : -109714417627648 : ffff9c371f1c4e00) 00000100:00000040:0.0:1788572469.869504:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c4e00 x1875440809929728/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.869510:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572469.869512:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572469.869515:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572469.869517:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809929728, offset 0 00000400:00000200:0.0:1788572469.869521:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572469.869525:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.869540:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572469.869544:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572469.869546:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572469.869549:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572469.869551:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572469.869554:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539644:539644:256:4294967295] 192.168.202.148@tcp LPNI seq info [539609:539609:8:4294967295] 00000400:00000200:0.0:1788572469.869558:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.869565:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572469.869569:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572469.869573:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572469.869577:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572469.869581:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572469.869583:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572469.869598:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.869603:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 173025 usecs (174 jiffies) 00000400:00000200:0.0:1788572469.869606:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572469.869608:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609049 00000100:00000001:0.0:1788572469.869610:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.869613:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609049 00000100:00000001:0.0:1788572469.869615:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572469.869616:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572469.869618:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.869620:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.869623:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572469.874413:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.874431:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.874436:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.874438:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.874447:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.874455:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.874463:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.874469:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.874474:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.874478:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41745ec00 00000400:00000200:0.0:1788572469.874483:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.874489:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529819 [1] + 224 00000800:00000001:0.0:1788572469.874496:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.874503:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.874507:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.874511:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.874519:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.874528:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4e00 x1875440809929728/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572469.874711:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c4e00 x1875440809929728/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.874727:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.874737:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.874741:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.908693:0:261278:0:(statahead.c:1053:sa_statahead()) Process entered 00000080:00000001:0.0:1788572469.908697:0:261278:0:(statahead.c:199:sa_alloc()) Process entered 00000080:00000010:0.0:1788572469.908701:0:261278:0:(statahead.c:203:sa_alloc()) kmalloced '(entry)': 120 at ffff9c372d676980. 00000020:00000001:0.0:1788572469.908704:0:261278:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc193fcd8 00000080:00400000:0.0:1788572469.908706:0:261278:0:(statahead.c:208:sa_alloc()) alloc sa entry f434.sanity-013(ffff9c372d676980) index 9 00000080:00000001:0.0:1788572469.908709:0:261278:0:(statahead.c:233:sa_alloc()) Process leaving (rc=18446634359531727232 : -109714177824384 : ffff9c372d676980) 00000080:00000001:0.0:1788572469.908712:0:261278:0:(statahead.c:977:sa_lookup()) Process entered 00000080:00000010:0.0:1788572469.908715:0:261278:0:(statahead.c:443:sa_prep_data()) kmalloced '(item)': 672 at ffff9c372d20f800. 00800000:00000002:0.0:1788572469.908720:0:261278:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 11 00800000:00000001:0.0:1788572469.908725:0:261278:0:(lmv_obd.c:4625:lmv_batch_add()) Process entered 00800000:00000001:0.0:1788572469.908727:0:261278:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572469.908729:0:261278:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572469.908730:0:261278:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572469.908732:0:261278:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572469.908734:0:261278:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572469.908735:0:261278:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572469.908737:0:261278:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908739:0:261278:0:(lmv_obd.c:4588:lmv_batch_get_sub()) Process entered 00800000:00000001:0.0:1788572469.908741:0:261278:0:(lmv_obd.c:4592:lmv_batch_get_sub()) Process leaving (rc=18446634359525744576 : -109714183807040 : ffff9c372d0c1fc0) 00000002:00000001:0.0:1788572469.908744:0:261278:0:(mdc_batch.c:198:mdc_batch_add()) Process entered 00000002:00000010:0.0:1788572469.908747:0:261278:0:(mdc_batch.c:207:mdc_batch_add()) kmalloced '(item->mop_pill)': 144 at ffff9c371eeb5a80. 00000002:00000001:0.0:1788572469.908749:0:261278:0:(batch.c:710:cli_batch_add()) Process entered 00000002:00000010:0.0:1788572469.908752:0:261278:0:(batch.c:289:batch_update_request_create()) kmalloced '(head)': 80 at ffff9c3711626e40. 00000002:00000010:0.0:1788572469.908756:0:261278:0:(batch.c:217:batch_update_buffer_create()) kmalloced '(buf)': 40 at ffff9c372d091bc0. 00000002:00000010:0.0:1788572469.908760:0:261278:0:(batch.c:223:batch_update_buffer_create()) kmalloced '(bur)': 4096 at ffff9c3711c59000. 00000002:00000001:0.0:1788572469.908762:0:261278:0:(batch.c:495:batch_update_request_add()) Process entered 00000002:00000001:0.0:1788572469.908764:0:261278:0:(mdc_batch.c:75:mdc_batch_getattr_pack()) Process entered 00000002:00000001:0.0:1788572469.908770:0:261278:0:(mdc_batch.c:32:mdc_ldlm_lock_pack()) Process entered 00010000:00000001:0.0:1788572469.908772:0:261278:0:(ldlm_request.c:1255:ldlm_cli_lock_create_pack()) Process entered 00010000:00000001:0.0:1788572469.908774:0:261278:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572469.908777:0:261278:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 11 00010000:00000001:0.0:1788572469.908779:0:261278:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.908781:0:261278:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572469.908784:0:261278:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b800. 00000020:00000001:0.0:1788572469.908787:0:261278:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572469.908788:0:261278:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b800 with handle 0x14415860931682bd to hash 00000020:00000001:0.0:1788572469.908790:0:261278:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572469.908792:0:261278:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553701888 : -109714155849728 : ffff9c372eb6b800) 00010000:00000001:0.0:1788572469.908794:0:261278:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553701888 : -109714155849728 : ffff9c372eb6b800) 00010000:00000001:0.0:1788572469.908796:0:261278:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.908798:0:261278:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.908801:0:261278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b800/0x14415860931682bd lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 12 type: IBT flags: 0x10000000000000 pid: 261278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.908808:0:261278:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b800/0x14415860931682bd lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x3/0x0 rrc: 12 type: IBT flags: 0x0 pid: 261278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.908813:0:261278:0:(ldlm_request.c:1287:ldlm_cli_lock_create_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908815:0:261278:0:(mdc_batch.c:51:mdc_ldlm_lock_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908818:0:261278:0:(mdc_batch.c:150:mdc_batch_getattr_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:0.0:1788572469.908822:0:261278:0:(batch.c:263:batch_insert_update_callback()) kmalloced '(ouc)': 40 at ffff9c372d091ec0. 00000002:00000001:0.0:1788572469.908824:0:261278:0:(batch.c:544:batch_update_request_add()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908826:0:261278:0:(batch.c:722:cli_batch_add()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908828:0:261278:0:(mdc_batch.c:213:mdc_batch_add()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908830:0:261278:0:(lmv_obd.c:4637:lmv_batch_add()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.908832:0:261278:0:(statahead.c:987:sa_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908835:0:261278:0:(lmv_obd.c:4492:lmv_batch_flush()) Process entered 00000002:00000001:0.0:1788572469.908836:0:261278:0:(batch.c:690:cli_batch_flush()) Process entered 00000002:00000001:0.0:1788572469.908838:0:261278:0:(batch.c:435:batch_send_update_req()) Process entered 00000002:00000001:0.0:1788572469.908839:0:261278:0:(batch.c:121:batch_prep_update_req()) Process entered 00000100:00000010:0.0:1788572469.908842:0:261278:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572469.908845:0:261278:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=45 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572469.908848:0:261278:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.908849:0:261278:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.908851:0:261278:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.908856:0:261278:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572469.908858:0:261278:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908861:0:261278:0:(batch.c:105:batch_prep_inline_update_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908863:0:261278:0:(batch.c:143:batch_prep_update_req()) Process leaving via out_req (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572469.908866:0:261278:0:(batch.c:198:batch_prep_update_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.908869:0:261278:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.908885:0:261278:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.908892:0:261278:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9424621] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b7100 x1875440809930112/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00000001:0.0:1788572469.908922:0:261278:0:(batch.c:480:batch_send_update_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572469.908924:0:261278:0:(batch.c:699:cli_batch_flush()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908926:0:261278:0:(lmv_obd.c:4514:lmv_batch_flush()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.908928:0:261278:0:(statahead.c:1081:sa_statahead()) Process leaving 00000080:00000010:0.0:1788572469.908930:0:261278:0:(statahead.c:1525:ll_statahead_by_fname()) kfreed 'fname': 255 at ffff9c37185d5000. 00000080:00000001:0.0:1788572469.908934:0:261278:0:(statahead.c:1526:ll_statahead_by_fname()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908936:0:261278:0:(lmv_obd.c:4492:lmv_batch_flush()) Process entered 00000002:00000001:0.0:1788572469.908938:0:261278:0:(batch.c:690:cli_batch_flush()) Process entered 00000002:00000001:0.0:1788572469.908939:0:261278:0:(batch.c:694:cli_batch_flush()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572469.908941:0:261278:0:(lmv_obd.c:4514:lmv_batch_flush()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.908942:0:261278:0:(statahead.c:1604:ll_statahead_thread()) Process leaving 00800000:00000001:0.0:1788572469.908944:0:261278:0:(lmv_obd.c:4459:lmv_batch_stop()) Process entered 00000002:00000001:0.0:1788572469.908946:0:261278:0:(batch.c:675:cli_batch_stop()) Process entered 00000002:00000001:0.0:1788572469.908947:0:261278:0:(batch.c:435:batch_send_update_req()) Process entered 00000002:00000001:0.0:1788572469.908948:0:261278:0:(batch.c:438:batch_send_update_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:0.0:1788572469.908949:0:261278:0:(batch.c:680:cli_batch_stop()) kfreed 'cbh': 32 at ffff9c372d0c1fc0. 00000002:00000001:0.0:1788572469.908951:0:261278:0:(batch.c:681:cli_batch_stop()) Process leaving (rc=0 : 0 : 0) 00800000:00000010:0.0:1788572469.908953:0:261278:0:(lmv_obd.c:4472:lmv_batch_stop()) kfreed 'sub': 32 at ffff9c372d0c1460. 00800000:00000010:0.0:1788572469.908955:0:261278:0:(lmv_obd.c:4480:lmv_batch_stop()) kfreed 'lbh': 48 at ffff9c3707985b00. 00800000:00000001:0.0:1788572469.908957:0:261278:0:(lmv_obd.c:4481:lmv_batch_stop()) Process leaving (rc=0 : 0 : 0) 00000080:00400000:0.0:1788572469.908959:0:261278:0:(statahead.c:1138:ll_stop_agl()) stop agl thread: sai ffff9c371985de00 pid 261279 00000800:00000200:0.0:1788572469.914244:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.914249:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572469.914255:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.914263:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.914270:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572469.914273:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572469.914277:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.914284:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7100 x1875440809930112/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572554 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.914295:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.914299:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7100 x1875440809930112/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572554 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.914310:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.914311:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572469.914384:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.914387:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572469.914391:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.914394:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.914397:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788572469.914399:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788572469.914402:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.914406:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4e00 x1875440809929984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.914412:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.914415:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4e00 x1875440809929984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.914421:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.914423:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000080:00400000:0.0:1788572469.914438:0:261278:0:(statahead.c:1153:ll_stop_agl()) agl thread stopped: sai ffff9c371985de00, parent d434.sanity 00000800:00000001:0.0:1788572469.932011:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.932029:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.932033:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.932035:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.932045:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.932053:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.932062:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.932067:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.932072:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.932076:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 3816 into portal 4 MB=0x6a9b41745ec80 00000400:00000200:0.0:1788572469.932081:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.932087:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 3816/3816 into md 0x529811 [4] + 224 00000800:00000001:0.0:1788572469.932093:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.932109:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.932113:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.932117:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.932124:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.932133:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70a80 x1875440809929856/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3472/7976 e 0 to 0 dl 1788572554 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572469.932143:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=3816 offset=224 replen=7976 req@ffff9c371ec70a80 x1875440809929856/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3472/7976 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.932165:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.932174:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.932177:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.938264:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938282:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.938286:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938288:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.938297:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938304:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.938313:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938318:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938323:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938327:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 680 into portal 4 MB=0x6a9b41745ed80 00000400:00000200:0.0:1788572469.938332:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938338:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 680/680 into md 0x529839 [2] + 224 00000800:00000001:0.0:1788572469.938345:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938350:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.938354:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.938358:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.938365:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.938373:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7100 x1875440809930112/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572554 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572469.938383:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=680 offset=224 replen=4320 req@ffff9c372d6b7100 x1875440809930112/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.938412:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.938419:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938423:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.938425:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938427:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.938430:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938433:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.938438:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938440:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938443:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938446:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b41745ed00 00000400:00000200:0.0:1788572469.938449:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.938454:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529829 [1] + 224 00000800:00000001:0.0:1788572469.938457:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938471:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.938473:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.938476:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.938479:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.938482:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4e00 x1875440809929984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572469.938490:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4e00 x1875440809929984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.938512:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.938517:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.938520:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572469.952352:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572469.952357:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952362:0:237084:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572469.952365:0:237084:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572469.952367:0:237084:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572469.952373:0:237084:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.952384:0:237084:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c373204b200 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572469.952388:0:237084:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572469.952392:0:237084:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.952398:0:237084:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572469.952400:0:237084:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572469.952404:0:237084:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.952410:0:237084:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572469.952413:0:237084:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572469.952414:0:237084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.952416:0:237084:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.952418:0:237084:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b200 refcount=3 00000020:00000001:0.0:1788572469.952420:0:237084:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609143808 : -109714100407808 : ffff9c373204b200) 00010000:00000001:0.0:1788572469.952422:0:237084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609143808 : -109714100407808 : ffff9c373204b200) 00010000:00000001:0.0:1788572469.952425:0:237084:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572469.952428:0:237084:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572469.952434:0:237084:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572469.952438:0:237084:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572469.952440:0:237084:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572469.952442:0:237084:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xd99:0x0] mode (null) 00000002:00000001:0.0:1788572469.952445:0:237084:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572469.952448:0:237084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572469.952454:0:237084:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25300 count: 4 00010000:00000001:0.0:1788572469.952457:0:237084:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.952461:0:237084:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572469.952465:0:237084:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572469.952468:0:237084:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572469.952474:0:237084:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952478:0:237084:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372e909c00/0x1441586093168277 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c5c expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00000001:0.0:1788572469.952484:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.952485:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572469.952487:0:237084:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572469.952488:0:237084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572469.952490:0:237084:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572469.952491:0:237084:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372e909c00 refcount=2 00000020:00000001:0.0:1788572469.952493:0:237084:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359551204352 : -109714158347264 : ffff9c372e909c00) 00010000:00000001:0.0:1788572469.952495:0:237084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359551204352 : -109714158347264 : ffff9c372e909c00) 00010000:00000001:0.0:1788572469.952497:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.952498:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572469.952499:0:237084:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572469.952501:0:237084:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572469.952502:0:237084:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.952505:0:237084:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572469.952508:0:237084:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572469.952509:0:237084:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572469.952511:0:237084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572469.952513:0:237084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572469.952515:0:237084:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204b200 with handle 0x14415860931682b6 from hash 00010000:00000001:0.0:1788572469.952518:0:237084:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572469.952520:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.952521:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.952522:0:237084:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572469.952524:0:237084:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572469.952526:0:237084:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572469.952531:0:237084:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572469.952533:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.952535:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.952536:0:237084:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.952538:0:237084:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572469.952541:0:237084:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572469.952549:0:237084:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572469.952553:0:237084:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=44 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572469.952558:0:237084:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572469.952560:0:237084:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572469.952562:0:237084:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572469.952569:0:237084:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7c00. 00000100:00000001:0.0:1788572469.952575:0:237084:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.952578:0:237084:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572469.952582:0:237084:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572469.952588:0:237084:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572469.952590:0:237084:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572469.952594:0:237084:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572469.952598:0:237084:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572469.952609:0:237084:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4e00 x1875440809930368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572469.952632:0:237084:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572469.952648:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572469.952650:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.952651:0:237084:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572469.952653:0:237084:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572469.952655:0:237084:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952658:0:237084:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572469.952663:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572469.952666:0:237084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x14415860931682b6 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xd99:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1d51 expref: -99 pid: 261280 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572469.952672:0:237084:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372eccb240. 00010000:00000001:0.0:1788572469.952682:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572469.952684:0:237084:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572469.952686:0:237084:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c372d530780. 00010000:00000001:0.0:1788572469.952688:0:237084:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572469.952690:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572469.952691:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952693:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572469.952694:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572469.952695:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952696:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572469.952698:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572469.952699:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572469.952701:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572469.953276:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.953281:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572469.953287:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.953291:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.953297:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572469.953300:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572469.953304:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.953308:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4e00 x1875440809930368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.953318:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.953322:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440809930368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.953328:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.953330:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572469.955823:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.955829:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.955832:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.955834:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.955840:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.955845:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.955851:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.955854:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.955857:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.955861:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41745ee80 00000400:00000200:0.0:1788572469.955865:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.955869:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529851 [1] + 224 00000800:00000001:0.0:1788572469.955885:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.955931:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.955935:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.955938:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.955942:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.955948:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4e00 x1875440809930368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572485 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572469.955958:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4e00 x1875440809930368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572469.955969:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.955974:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.955977:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572469.989079:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572469.989089:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572469.989096:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.989104:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572469.989111:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3dd0 00000400:00000010:0.0:1788572469.989114:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3dd0. 00000100:00000001:0.0:1788572469.989119:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572469.989127:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7800 x1875440809930496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.989139:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572469.989142:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7800 x1875440809930496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.989149:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572469.989151:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572469.997449:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.997470:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572469.997476:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.997479:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.997489:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.997497:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572469.997508:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.997513:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.997519:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.997524:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b41745ef00 00000400:00000200:0.0:1788572469.997529:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572469.997535:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529861 [1] + 224 00000800:00000001:0.0:1788572469.997543:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.997561:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572469.997565:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572469.997571:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572469.997581:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572469.997591:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7800 x1875440809930496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572469.997602:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c7800 x1875440809930496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572554 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572469.997622:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572469.997630:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572469.997663:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.000017:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.000021:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.000126:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.000132:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572470.000136:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.000142:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.000147:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572470.000149:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572470.000153:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.000157:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7800 x1875440809930624/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.000167:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.000170:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440809930624/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.000177:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.000179:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.007180:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.007198:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.007203:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.007206:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.007216:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.007223:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.007233:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.007239:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.007244:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.007249:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41745ef80 00000400:00000200:0.0:1788572470.007254:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.007260:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529871 [1] + 224 00000800:00000001:0.0:1788572470.007267:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.007276:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.007280:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.007284:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.007292:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.007301:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7800 x1875440809930624/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.007311:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c7800 x1875440809930624/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572485 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.007331:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.007338:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.007341:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.038339:0:261284:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.038781:0:261284:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.038784:0:261284:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.038789:0:261284:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2738. 00000400:00000200:0.0:1788572470.038794:0:261284:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572470.038797:0:261284:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572470.038800:0:261284:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.038816:0:261284:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.038825:0:261284:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.038832:0:261284:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.038834:0:261284:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.038837:0:261284:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.038840:0:261284:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.038842:0:261284:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.038844:0:261284:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.038847:0:261284:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.038858:0:261284:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=34 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572470.038861:0:261284:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.038864:0:261284:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.038866:0:261284:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.038868:0:261284:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.038881:0:261284:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.038885:0:261284:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.038887:0:261284:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.038889:0:261284:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.038892:0:261284:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.038896:0:261284:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.038903:0:261284:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.038906:0:261284:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.038908:0:261284:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.038912:0:261284:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.038919:0:261284:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261284:1875440809930880:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.038924:0:261284:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.038927:0:261284:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.038930:0:261284:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572470.038932:0:261284:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.038936:0:261284:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.038942:0:261284:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572470.038944:0:261284:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c372d946b40. 00000100:00000001:0.0:1788572470.038949:0:261284:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572470.038950:0:261284:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.038953:0:261284:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.038957:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.038960:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.038964:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=4 00000020:00000001:0.0:1788572470.038966:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.038969:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000040:0.0:1788572470.038975:0:261284:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204b800, remote cookie: 0xd9894672d99a1d9e, flags: 0x1001 00010000:00010000:0.0:1788572470.038980:0:261284:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.038989:0:261284:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xd9b:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572470.038992:0:261284:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572470.038998:0:261284:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25400 count: 3 00010000:00000001:0.0:1788572470.039001:0:261284:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.039004:0:261284:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.039008:0:261284:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.039015:0:261284:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039021:0:261284:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.039025:0:261284:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572470.039027:0:261284:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572470.039029:0:261284:0:(ldlm_lock.c:1024:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572470.039031:0:261284:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572470.039033:0:261284:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xd9b:0x0].0x0 (ffff9c3730c25400) refcount = 3 00010000:00000040:0.0:1788572470.039036:0:261284:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.039040:0:261284:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 1/0,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000040:0.0:1788572470.039047:0:261284:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731f15000/0x1441586093167d29 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a09e7 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572470.039053:0:261284:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039059:0:261284:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572470.039062:0:261284:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.039064:0:261284:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.039081:0:261284:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.039083:0:261284:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.039086:0:261284:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039092:0:261284:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.039093:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039095:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039096:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039097:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039098:0:261284:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.039102:0:261284:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572470.039104:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039105:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039107:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=3 00000020:00000001:0.0:1788572470.039108:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039110:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039113:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039114:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039116:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=4 00000020:00000001:0.0:1788572470.039117:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039119:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039121:0:261284:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.039124:0:261284:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.039127:0:261284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039134:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039135:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039137:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039138:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039139:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=5 00000020:00000001:0.0:1788572470.039141:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039143:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039145:0:261284:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.039148:0:261284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039153:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039154:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.039156:0:261284:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039162:0:261284:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.039163:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039164:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039165:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039166:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572470.039171:0:261284:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572470.039185:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039186:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039187:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=3 00000020:00000001:0.0:1788572470.039189:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039191:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00000002:00010000:0.0:1788572470.039195:0:261284:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572470.039204:0:261284:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091540. 00010000:00000001:0.0:1788572470.039206:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039207:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.039209:0:261284:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.039210:0:261284:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.039213:0:261284:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572470.039216:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039217:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039218:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=3 00000020:00000001:0.0:1788572470.039220:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039222:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00000002:00010000:0.0:1788572470.039225:0:261284:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039231:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039231:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039233:0:261284:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572470.039235:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039236:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039237:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=3 00000020:00000001:0.0:1788572470.039239:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039240:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000040:0.0:1788572470.039243:0:261284:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25400 count: 4 00010000:00000001:0.0:1788572470.039246:0:261284:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.039249:0:261284:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.039251:0:261284:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.039254:0:261284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 02000000:00000001:0.0:1788572470.039262:0:261284:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.039267:0:261284:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.039270:0:261284:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.039275:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039276:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039278:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039279:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039280:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=3 00000020:00000001:0.0:1788572470.039281:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039283:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.039285:0:261284:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:0.0:1788572470.039288:0:261284:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039294:0:261284:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.039296:0:261284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039302:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039303:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.039305:0:261284:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b800/0x14415860931682d9 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a1d9e expref: -99 pid: 261284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.039311:0:261284:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.039313:0:261284:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.039317:0:261284:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:0.0:1788572470.039321:0:261284:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff9c371eeb5cc0. 00010000:00000001:0.0:1788572470.039324:0:261284:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:0.0:1788572470.039440:0:261284:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:0.0:1788572470.039443:0:261284:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.039445:0:261284:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.039446:0:261284:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.039448:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039449:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.039450:0:261284:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572470.039453:0:261284:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7710 00000002:00002000:0.0:1788572470.039455:0:261284:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-007 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572470.039458:0:261284:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.039463:0:261284:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.039466:0:261284:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572470.039469:0:261284:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572470.039472:0:261284:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.039475:0:261284:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.039479:0:261284:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572470.039481:0:261284:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1ed7b20 it_disposition b 00000080:00000001:0.0:1788572470.039484:0:261284:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572470.039491:0:261284:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572470.039494:0:261284:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572470.039496:0:261284:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329051 : 144115373145329051 : 200002b17000d9b) 00000080:00000001:0.0:1788572470.039499:0:261284:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572470.039506:0:261284:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.039510:0:261284:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572470.039513:0:261284:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572470.039516:0:261284:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.039520:0:261284:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.039523:0:261284:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.039530:0:261284:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.039535:0:261284:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.039540:0:261284:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572470.039542:0:261284:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572470.039545:0:261284:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xd9b:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.039550:0:261284:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.039554:0:261284:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20972 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.039557:0:261284:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711ace000) owner ffff9c372d1c0000 00020000:00000001:0.0:1788572470.039560:0:261284:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.039563:0:261284:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711ace000) owner ffff9c372d1c0000 00020000:00000010:0.0:1788572470.039567:0:261284:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.039570:0:261284:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.039573:0:261284:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572470.039575:0:261284:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xd9b:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.039577:0:261284:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.039579:0:261284:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572470.039581:0:261284:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572470.039586:0:261284:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329051 : 144115373145329051 : 200002b17000d9b) 00000080:00000001:0.0:1788572470.039588:0:261284:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572470.039594:0:261284:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xd9b:0x0](ffff9c372f0f8908): rc = 0 00000080:00000001:0.0:1788572470.039596:0:261284:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359559522568 : -109714150029048 : ffff9c372f0f8908) 00010000:00000001:0.0:1788572470.039598:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039600:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039602:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee800 refcount=3 00000020:00000001:0.0:1788572470.039603:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.039605:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00000080:00000001:0.0:1788572470.039609:0:261284:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572470.039610:0:261284:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.039611:0:261284:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572470.039613:0:261284:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572470.039615:0:261284:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.039617:0:261284:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.039618:0:261284:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.039621:0:261284:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.039623:0:261284:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.039625:0:261284:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572470.039627:0:261284:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572470.039629:0:261284:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xd9b:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.039633:0:261284:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.039658:0:261284:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20972 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.039661:0:261284:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711ace000) owner ffff9c372d1c0000 00020000:00000001:0.0:1788572470.039663:0:261284:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.039664:0:261284:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711ace000) owner ffff9c372d1c0000 00020000:00000010:0.0:1788572470.039667:0:261284:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.039670:0:261284:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.039672:0:261284:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572470.039674:0:261284:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xd9b:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.039675:0:261284:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.039677:0:261284:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.039680:0:261284:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572470.039683:0:261284:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711ace000) owner 0000000000000000/ffff9c372d1c0000 00020000:00000002:0.0:1788572470.039686:0:261284:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3711bddc00 addref 2/0 by ffff9c372d1c0000. 00020000:00000002:0.0:1788572470.039689:0:261284:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711ace000) owner 0000000000000000/ffff9c372d1c0000 00020000:00000001:0.0:1788572470.039692:0:261284:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572470.039694:0:261284:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572470.039696:0:261284:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572470.039697:0:261284:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572470.039699:0:261284:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572470.039702:0:261284:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xd9b:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572470.039704:0:261284:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572470.039706:0:261284:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.039708:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039709:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.039710:0:261284:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572470.039713:0:261284:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572470.039715:0:261284:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.039718:0:261284:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xd9b:0x0](ffff9c372f0f8908) for lock 0x1441586093168285 00800000:00000001:0.0:1788572470.039721:0:261284:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.039723:0:261284:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.039724:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039725:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039727:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee800 refcount=3 00000020:00000001:0.0:1788572470.039729:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.039732:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.039734:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039736:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.039737:0:261284:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.039739:0:261284:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572470.039746:0:261284:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37081410d8 inode ffff9c372f0f8908 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572470.039750:0:261284:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572470.039754:0:261284:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xd9b:0x0](ffff9c372f0f8908) 00000080:00010000:0.0:1788572470.039758:0:261284:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168285 from it ffffb95bc1ed7b20 00010000:00000001:0.0:1788572470.039760:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.039761:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.039762:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee800 refcount=3 00000020:00000001:0.0:1788572470.039764:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.039766:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.039767:0:261284:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.039770:0:261284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.039776:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039777:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.039778:0:261284:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.039781:0:261284:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.039783:0:261284:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00000001:0.0:1788572470.039790:0:261284:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.039793:0:261284:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.039795:0:261284:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.039801:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.039802:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.041010:0:261284:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041018:0:261284:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.041021:0:261284:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041023:0:261284:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041026:0:261284:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041030:0:261284:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.041031:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041033:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.041036:0:261284:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572470.041042:0:261284:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711215800. 00000100:00000001:0.0:1788572470.041050:0:261284:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.041072:0:261284:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440809930880/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.041081:0:261284:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.041084:0:261284:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.041087:0:261284:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c3705367800. 02000000:00000001:0.0:1788572470.041090:0:261284:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.041093:0:261284:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.041095:0:261284:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.041097:0:261284:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.041101:0:261284:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c3718ec3400. 02000000:00000001:0.0:1788572470.041104:0:261284:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.041106:0:261284:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.041108:0:261284:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788572470.041111:0:261284:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.041113:0:261284:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.041116:0:261284:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572470.041117:0:261284:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1ed7b20 released 00000080:00000001:0.0:1788572470.041118:0:261284:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572470.041129:0:261284:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7d08 00000080:00200008:0.0:1788572470.041131:0:261284:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-007:[0x200002b17:0xd9b:0x0](ffff9c372f0f8908), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.041138:0:261284:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.041139:0:261284:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7c80 00000080:00200000:0.0:1788572470.041141:0:261284:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xd9b:0x0](ffff9c372f0f8908),name=f434.sanity-007 00000080:00000010:0.0:1788572470.041148:0:261284:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572470.041153:0:261284:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.041160:0:261284:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.041163:0:261284:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.041166:0:261284:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7ba0 00800000:00000002:0.0:1788572470.041168:0:261284:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xd9b:0x0] 'f434.sanity-007' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.041171:0:261284:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.041176:0:261284:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.041178:0:261284:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.041182:0:261284:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.041184:0:261284:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.041186:0:261284:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.041188:0:261284:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.041191:0:261284:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.041193:0:261284:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.041194:0:261284:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.041195:0:261284:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.041196:0:261284:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.041197:0:261284:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.041199:0:261284:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd9b:0x0] 00800000:00000001:0.0:1788572470.041201:0:261284:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.041203:0:261284:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xd9b:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.041208:0:261284:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.041210:0:261284:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7a70 00000002:00010000:0.0:1788572470.041211:0:261284:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xd9b:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.041215:0:261284:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.041217:0:261284:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.041219:0:261284:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.041225:0:261284:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25400 count: 4 00010000:00000001:0.0:1788572470.041228:0:261284:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.041232:0:261284:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.041236:0:261284:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.041240:0:261284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 02000000:00000001:0.0:1788572470.041249:0:261284:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.041252:0:261284:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.041256:0:261284:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00000001:0.0:1788572470.041267:0:261284:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.041270:0:261284:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.041272:0:261284:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.041274:0:261284:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.041275:0:261284:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.041278:0:261284:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572470.041280:0:261284:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.041282:0:261284:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572470.041285:0:261284:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572470.041290:0:261284:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.041292:0:261284:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.041294:0:261284:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xd9b:0x0](ffff9c372f0f8908) 00000080:00010000:0.0:1788572470.041297:0:261284:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xd9b:0x0](ffff9c372f0f8908) for lock 0x1441586093168285 00800000:00000001:0.0:1788572470.041300:0:261284:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.041302:0:261284:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.041303:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.041305:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.041307:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee800 refcount=3 00000020:00000001:0.0:1788572470.041309:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.041312:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.041315:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041316:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.041317:0:261284:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.041319:0:261284:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.041321:0:261284:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168285 from it ffffb95bc1ed7cb8 00010000:00000001:0.0:1788572470.041323:0:261284:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.041324:0:261284:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.041325:0:261284:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee800 refcount=3 00000020:00000001:0.0:1788572470.041327:0:261284:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.041329:0:261284:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609812992 : -109714099738624 : ffff9c37320ee800) 00010000:00000001:0.0:1788572470.041331:0:261284:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.041335:0:261284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00000001:0.0:1788572470.041341:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041342:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.041343:0:261284:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.041345:0:261284:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.041347:0:261284:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee800/0x1441586093168285 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1c94 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00000001:0.0:1788572470.041353:0:261284:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.041355:0:261284:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.041357:0:261284:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.041360:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041361:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.041362:0:261284:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041364:0:261284:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.041365:0:261284:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041367:0:261284:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041369:0:261284:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041371:0:261284:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.041372:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041373:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.041375:0:261284:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.041376:0:261284:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.041379:0:261284:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572470.041381:0:261284:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572470.041383:0:261284:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.041385:0:261284:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572470.041387:0:261284:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.041389:0:261284:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572470.041391:0:261284:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572470.041393:0:261284:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.041397:0:261284:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572470.041399:0:261284:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572470.041403:0:261284:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1080 -> ffff9c37313e1080 00000020:00000001:0.0:1788572470.041406:0:261284:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572470.041408:0:261284:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572470.041410:0:261284:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd9b:0x0] ignore/verify layout 0/1, layout version 2 restore needed 0 00000020:00000001:0.0:1788572470.041413:0:261284:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.041415:0:261284:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572470.041417:0:261284:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.041419:0:261284:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.041421:0:261284:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.041424:0:261284:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd9b:0x0]io ffff9c371ff53590 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572470.041427:0:261284:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711ace000) owner 0000000000000000/ffff9c372d1c0000 00020000:00000001:0.0:1788572470.041431:0:261284:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572470.041433:0:261284:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572470.041435:0:261284:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572470.041437:0:261284:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041439:0:261284:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572470.041442:0:261284:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572470.041443:0:261284:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041445:0:261284:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.041445:0:261284:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572470.041447:0:261284:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572470.041449:0:261284:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711ace000) owner 0000000000000000/ffff9c372d1c0000 00000020:00000001:0.0:1788572470.041451:0:261284:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572470.041453:0:261284:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572470.041455:0:261284:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xd9b:0x0] 00000020:00000001:0.0:1788572470.041457:0:261284:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572470.041459:0:261284:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572470.041461:0:261284:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572470.041462:0:261284:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572470.041468:0:261284:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572470.041473:0:261284:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676600. 00020000:00000002:0.0:1788572470.041476:0:261284:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572470.041478:0:261284:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572470.041480:0:261284:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.041481:0:261284:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572470.041483:0:261284:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.041485:0:261284:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359600765624 : -109714108785992 : ffff9c373184dab8) 00000020:00001000:0.0:1788572470.041487:0:261284:0:(cl_object.c:823:cl_env_get()) 1@ffff9c373184dab0 00000020:00000001:0.0:1788572470.041489:0:261284:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1918 -> ffff9c37313e1080 00000020:00000001:0.0:1788572470.041491:0:261284:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572470.041495:0:261284:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.041496:0:261284:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572470.041497:0:261284:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041499:0:261284:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041501:0:261284:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293051568 : -109714416500048 : ffff9c371f2d82b0) 00000020:00000001:0.0:1788572470.041503:0:261284:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572470.041507:0:261284:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6be0. 00000008:00010000:0.0:1788572470.041510:0:261284:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572470.041512:0:261284:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572470.041514:0:261284:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572470.041515:0:261284:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676630, osc lock ffff9c37030a6be0, flags 0x400001002 00000020:00000001:0.0:1788572470.041517:0:261284:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041519:0:261284:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041521:0:261284:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531726336 : -109714177825280 : ffff9c372d676600) 00000020:00000001:0.0:1788572470.041523:0:261284:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572470.041524:0:261284:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572470.041526:0:261284:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041527:0:261284:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041530:0:261284:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572470.041532:0:261284:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572470.041533:0:261284:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572470.041535:0:261284:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572470.041536:0:261284:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.041537:0:261284:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293051568 : -109714416500048 : ffff9c371f2d82b0) 00000020:00000001:0.0:1788572470.041539:0:261284:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572470.041541:0:261284:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572470.041544:0:261284:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572470.041548:0:261284:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572470.041550:0:261284:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572470.041551:0:261284:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572470.041553:0:261284:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572470.041554:0:261284:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572470.041555:0:261284:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572470.041558:0:261284:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572470.041559:0:261284:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572470.041563:0:261284:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041566:0:261284:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572470.041570:0:261284:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572470.041573:0:261284:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572470.041577:0:261284:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.041582:0:261284:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.041585:0:261284:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.041586:0:261284:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.041587:0:261284:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.041590:0:261284:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.041592:0:261284:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.041595:0:261284:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.041600:0:261284:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7e00. 00000100:00000001:0.0:1788572470.041606:0:261284:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041610:0:261284:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.041612:0:261284:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041613:0:261284:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.041615:0:261284:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041617:0:261284:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.041619:0:261284:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572470.041622:0:261284:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.041624:0:261284:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572470.041628:0:261284:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76800. 00010000:00000010:0.0:1788572470.041632:0:261284:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3708c76000. 00010000:00000001:0.0:1788572470.041664:0:261284:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.041668:0:261284:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6a000. 00000020:00000001:0.0:1788572470.041672:0:261284:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.041674:0:261284:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6a000 with handle 0x14415860931682e0 to hash 00000020:00000001:0.0:1788572470.041676:0:261284:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.041677:0:261284:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000010:0.0:1788572470.041681:0:261284:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091e00. 00010000:00000001:0.0:1788572470.041684:0:261284:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.041687:0:261284:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.041688:0:261284:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.041692:0:261284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.041702:0:261284:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.041713:0:261284:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.041719:0:261284:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.041723:0:261284:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.041733:0:261284:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572470.041770:0:261284:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572470.041773:0:261284:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041775:0:261284:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.041777:0:261284:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041779:0:261284:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.041782:0:261284:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.041784:0:261284:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572470.041786:0:261284:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572470.041787:0:261284:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572470.041797:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572470.041801:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572470.041803:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.042005:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.042008:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.042010:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.042012:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.042015:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.042021:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.042030:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.042032:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.042035:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.042037:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.042040:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.042043:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809931008:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.042047:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.042053:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.042055:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.042058:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.042234:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788572470.042238:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.042246:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.042251:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b00. 00000400:00000010:0.0:1788572470.042258:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572470.042265:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809931008, portal 4 00000100:00000001:0.0:1788572470.042267:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.042269:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572470.042275:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.042282:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.042284:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.042287:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572470.042290:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809931008, offset 0 00000400:00000200:0.0:1788572470.042296:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.042304:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.042311:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.042317:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.042321:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.042324:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.042327:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.042331:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539653:539653:256:4294967295] 192.168.202.148@tcp LPNI seq info [539618:539618:8:4294967295] 00000400:00000200:0.0:1788572470.042336:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.042344:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.042350:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.042355:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572470.042361:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.042369:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.042373:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.042503:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.042508:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 183060 usecs (184 jiffies) 00000400:00000200:0.0:1788572470.042512:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.042514:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609232 00000100:00000001:0.0:1788572470.042517:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609316 00000100:00000001:0.0:1788572470.042520:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609232 00000100:00000001:0.0:1788572470.042522:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609316 00000100:00000001:0.0:1788572470.042524:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.042527:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.042529:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.042532:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.049104:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.049109:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.049116:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.049118:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.049122:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8b00. 00000400:00000200:0.0:1788572470.049126:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572470.049129:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572470.049132:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.049137:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.049145:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.049152:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.049153:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.049156:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.049159:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.049162:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.049164:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.049167:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.049177:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=34 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.049181:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=7 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572470.049184:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.049186:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049189:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.049193:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.049196:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.049199:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572470.049201:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.049204:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.049209:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.049217:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.049220:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.049223:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572470.049226:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572470.049229:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049232:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049235:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=4 00000020:00000001:0.0:1788572470.049237:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049240:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049243:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049244:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049245:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=5 00000020:00000001:0.0:1788572470.049246:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049249:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049252:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.049255:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.049260:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049271:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049273:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.049275:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.049277:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049278:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049279:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=6 00000020:00000001:0.0:1788572470.049282:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049284:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000040:0.0:1788572470.049291:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6a000, remote cookie: 0xd9894672d99a1da5, flags: 0x1001 00010000:00010000:0.0:1788572470.049296:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049304:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572470.049307:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049309:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.049313:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572470.049317:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.049326:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51ec:0x0].0x0 (ffff9c3708c76800) refcount = 1 00010000:00000040:0.0:1788572470.049328:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.049331:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049340:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.049341:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.049344:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.049346:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.049350:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049355:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.049357:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049358:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.049359:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049360:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.049362:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572470.049365:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572470.049368:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049369:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049370:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=5 00000020:00000001:0.0:1788572470.049372:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049374:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00000008:00000001:0.0:1788572470.049378:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572470.049383:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9c00 -> ffff9c37313e1080 00000008:00010000:0.0:1788572470.049387:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.049394:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9c00 -> ffff9c37313e1080 00000020:00000001:0.0:1788572470.049396:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572470.049399:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572470.049401:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572470.049404:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.049406:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.049407:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9c00 -> ffff9c37313e1080 00000008:00000001:0.0:1788572470.049409:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572470.049412:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.049414:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572470.049432:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572470.049434:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049436:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049438:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049439:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=6 00000020:00000001:0.0:1788572470.049441:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049443:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049445:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.049448:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049454:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049456:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.049458:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049464:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.049466:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049467:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.049468:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049470:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.049471:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.049472:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a000 refcount=5 00000020:00000001:0.0:1788572470.049474:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049476:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553695744 : -109714155855872 : ffff9c372eb6a000) 00010000:00000001:0.0:1788572470.049478:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.049480:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.049487:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049488:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.049489:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.049492:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.049494:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000001c70cead/0x14415860931682e0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a1da5 expref: -99 pid: 261284 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572470.049502:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.049505:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.049507:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.049511:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049512:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.049514:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049517:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.049519:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049521:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049523:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.049526:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.049527:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049528:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.049529:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.049530:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.049531:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.049537:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.049546:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809931008:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.049551:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.049554:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.049558:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809931008/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.049565:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.049567:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.049570:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788572470.049574:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.049577:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.049579:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572470.049581:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.049584:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7e00. 02000000:00000001:0.0:1788572470.049590:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.049592:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.049594:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572470.049597:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.049598:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.049601:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.049609:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.049610:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.049613:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.049615:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.084038:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.084043:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572470.084049:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.084054:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.084071:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1bb0 00000400:00000010:0.0:1788572470.084074:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1bb0. 00000100:00000001:0.0:1788572470.084078:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.084083:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809931392/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.084092:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.084095:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440809931392/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.084102:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.084104:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.085746:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.085750:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.089435:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.089451:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.089455:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.089457:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.089466:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.089473:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.089482:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.089486:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.089491:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.089495:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41745f300 00000400:00000200:0.0:1788572470.089500:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.089506:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x5298c1 [1] + 224 00000800:00000001:0.0:1788572470.089514:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.089521:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.089524:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.089529:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.089537:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.089548:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809931520/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.089572:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c6300 x1875440809931520/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.089593:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.089601:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.089604:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.095756:0:261286:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.095760:0:261286:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.095767:0:261286:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.095770:0:261286:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1530 -> ffff9c37313e1530 00000020:00000001:0.0:1788572470.095772:0:261286:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1530 -> ffff9c37313e1530 00000020:00000001:0.0:1788572470.095773:0:261286:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.095778:0:261286:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.095783:0:261286:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.095786:0:261286:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.095787:0:261286:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.095789:0:261286:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.095791:0:261286:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xd9c:11031 on OST[0]: s=0 (1) m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.095795:0:261286:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.095797:0:261286:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.095800:0:261286:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.095802:0:261286:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.095804:0:261286:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1530 -> ffff9c37313e1530 00000080:00200000:0.0:1788572470.095817:0:261286:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xd9c:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.095819:0:261286:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.095822:0:261286:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.095824:0:261286:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff514d0 (ffff9c3718da6a28/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.095827:0:261286:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.095828:0:261286:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff514d0 (ffff9c3718da6a28/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.095832:0:261286:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.095836:0:261286:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.095838:0:261286:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293061496 : -109714416490120 : ffff9c371f2da978) 00000020:00000001:0.0:1788572470.095839:0:261286:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.095840:0:261286:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cf430 (ffff9c3718da6f38/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.095844:0:261286:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.095845:0:261286:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.095847:0:261286:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.095849:0:261286:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.095850:0:261286:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.095852:0:261286:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.095853:0:261286:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.095854:0:261286:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.095856:0:261286:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.095857:0:261286:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff514d0 (ffff9c3718da6a28/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.095860:0:261286:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.095861:0:261286:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.095862:0:261286:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cf430 (ffff9c3718da6a28/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.095865:0:261286:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb9690. 00000020:00000001:0.0:1788572470.095868:0:261286:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.095870:0:261286:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cf400. 00020000:00000001:0.0:1788572470.095883:0:261286:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.095885:0:261286:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.095886:0:261286:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.095888:0:261286:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.095891:0:261286:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.095893:0:261286:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.095895:0:261286:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.095896:0:261286:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.095898:0:261286:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.095900:0:261286:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6f30 00020000:00000001:0.0:1788572470.095903:0:261286:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.095905:0:261286:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.095907:0:261286:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.095910:0:261286:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd9c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.095914:0:261286:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.095916:0:261286:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.095918:0:261286:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.095920:0:261286:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.095921:0:261286:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.095922:0:261286:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6a20 00000001:00000001:0.0:1788572470.095924:0:261286:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.095932:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bd08 00000080:00000008:0.0:1788572470.095935:0:261286:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-008:[0x200002b17:0xd9c:0x0](ffff9c372f0f80c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.095942:0:261286:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.095943:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bda0 00000080:00002000:0.0:1788572470.095945:0:261286:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-008 (ffff9c3719d931b0, parent ffff9c3719d93af8, inode ffff9c372f0f80c8) hashed, 00000080:00000001:0.0:1788572470.095948:0:261286:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.095980:0:261286:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.095984:0:261286:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.095986:0:261286:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.095987:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bbc0 00000080:00200000:0.0:1788572470.095989:0:261286:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.095996:0:261286:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572470.096002:0:261286:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.096006:0:261286:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.096010:0:261286:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.096012:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bae0 00800000:00000002:0.0:1788572470.096013:0:261286:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.096017:0:261286:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.096019:0:261286:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.096022:0:261286:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.096024:0:261286:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.096026:0:261286:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.096028:0:261286:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.096030:0:261286:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.096032:0:261286:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.096036:0:261286:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.096041:0:261286:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.096042:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b9b0 00000002:00010000:0.0:1788572470.096044:0:261286:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.096047:0:261286:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.096048:0:261286:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.096051:0:261286:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.096057:0:261286:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.096059:0:261286:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.096063:0:261286:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.096066:0:261286:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.096070:0:261286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 115 lru_type: 1 02000000:00000001:0.0:1788572470.096078:0:261286:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.096082:0:261286:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.096086:0:261286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 115 lru_type: 1 00000002:00000001:0.0:1788572470.096093:0:261286:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.096095:0:261286:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.096096:0:261286:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.096098:0:261286:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.096100:0:261286:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.096102:0:261286:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.096105:0:261286:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.096107:0:261286:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.096111:0:261286:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572470.096117:0:261286:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.096118:0:261286:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.096121:0:261286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.096124:0:261286:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.096127:0:261286:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.096129:0:261286:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.096130:0:261286:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.096131:0:261286:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.096133:0:261286:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.096135:0:261286:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.096137:0:261286:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.096140:0:261286:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.096141:0:261286:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.096142:0:261286:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.096144:0:261286:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.096146:0:261286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a0bbf8 00010000:00000001:0.0:1788572470.096148:0:261286:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.096149:0:261286:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.096151:0:261286:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.096152:0:261286:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.096154:0:261286:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.096156:0:261286:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.096159:0:261286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 115 lru_type: 1 00010000:00000001:0.0:1788572470.096166:0:261286:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.096167:0:261286:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.096168:0:261286:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.096170:0:261286:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.096172:0:261286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 116 lru_type: 1 00010000:00000001:0.0:1788572470.096178:0:261286:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.096180:0:261286:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.096181:0:261286:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.096183:0:261286:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.096184:0:261286:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.096185:0:261286:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.096187:0:261286:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.096188:0:261286:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.096190:0:261286:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.096192:0:261286:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.096193:0:261286:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.096194:0:261286:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.096196:0:261286:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.096197:0:261286:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.096199:0:261286:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.096201:0:261286:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.096204:0:261286:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.096207:0:261286:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.096209:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bb88 00000080:00002000:0.0:1788572470.096210:0:261286:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 8 00000080:00000001:0.0:1788572470.096213:0:261286:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.096215:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bbd0 00000080:00200000:0.0:1788572470.096216:0:261286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.096221:0:261286:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.096225:0:261286:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.096226:0:261286:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.096229:0:261286:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.096231:0:261286:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.096232:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bbe8 00000080:00002000:0.0:1788572470.096234:0:261286:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-008(ffff9c3719d931b0) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.096236:0:261286:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.096237:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bc30 00000080:00200000:0.0:1788572470.096238:0:261286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-008, flags=9 00000080:00000001:0.0:1788572470.096250:0:261286:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.096253:0:261286:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.096256:0:261286:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.096257:0:261286:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0be40 00000080:00002000:0.0:1788572470.096258:0:261286:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-008 (ffff9c3719d931b0, parent ffff9c3719d93af8, inode ffff9c372f0f80c8) hashed, 00000080:00000001:0.0:1788572470.096261:0:261286:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.136935:0:261288:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.136939:0:261288:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.136946:0:261288:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.136949:0:261288:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1e90 -> ffff9c37313e1e90 00000020:00000001:0.0:1788572470.136951:0:261288:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1e90 -> ffff9c37313e1e90 00000020:00000001:0.0:1788572470.136952:0:261288:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.136956:0:261288:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.136961:0:261288:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.136964:0:261288:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.136965:0:261288:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.136966:0:261288:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.136969:0:261288:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xd9d:11031 on OST[1]: s=0 (1) m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.136973:0:261288:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.136975:0:261288:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.136978:0:261288:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.136980:0:261288:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.136982:0:261288:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1e90 -> ffff9c37313e1e90 00000080:00200000:0.0:1788572470.136984:0:261288:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xd9d:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.136986:0:261288:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.136998:0:261288:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.137000:0:261288:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3705179970 (ffff9c3711a1d128/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.137003:0:261288:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.137005:0:261288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3705179970 (ffff9c3711a1d128/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.137009:0:261288:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.137012:0:261288:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.137014:0:261288:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859899552 : -109714849652064 : ffff9c37055c22a0) 00000020:00000001:0.0:1788572470.137016:0:261288:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.137017:0:261288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cfab0 (ffff9c3711a1dd88/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.137020:0:261288:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.137021:0:261288:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.137022:0:261288:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.137023:0:261288:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.137025:0:261288:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.137026:0:261288:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.137028:0:261288:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.137029:0:261288:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.137030:0:261288:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.137031:0:261288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3705179970 (ffff9c3711a1d128/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.137033:0:261288:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.137034:0:261288:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.137035:0:261288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cfab0 (ffff9c3711a1d128/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.137038:0:261288:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb8390. 00000020:00000001:0.0:1788572470.137041:0:261288:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.137043:0:261288:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cfa80. 00020000:00000001:0.0:1788572470.137045:0:261288:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.137046:0:261288:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.137047:0:261288:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.137049:0:261288:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.137052:0:261288:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.137055:0:261288:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.137056:0:261288:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.137057:0:261288:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.137059:0:261288:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.137062:0:261288:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dd80 00020000:00000001:0.0:1788572470.137064:0:261288:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.137066:0:261288:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.137069:0:261288:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.137071:0:261288:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd9d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.137075:0:261288:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.137077:0:261288:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.137079:0:261288:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.137081:0:261288:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.137083:0:261288:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.137084:0:261288:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d120 00000001:00000001:0.0:1788572470.137085:0:261288:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.137094:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87d08 00000080:00000008:0.0:1788572470.137097:0:261288:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-009:[0x200002b17:0xd9d:0x0](ffff9c372f0fec08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.137103:0:261288:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.137104:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87da0 00000080:00002000:0.0:1788572470.137106:0:261288:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-009 (ffff9c3708141000, parent ffff9c3719d93af8, inode ffff9c372f0fec08) hashed, 00000080:00000001:0.0:1788572470.137109:0:261288:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.137152:0:261288:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.137155:0:261288:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.137157:0:261288:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.137159:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87bc0 00000080:00200000:0.0:1788572470.137161:0:261288:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.137167:0:261288:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572470.137174:0:261288:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.137178:0:261288:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.137183:0:261288:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.137185:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87ae0 00800000:00000002:0.0:1788572470.137187:0:261288:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.137190:0:261288:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.137193:0:261288:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.137195:0:261288:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.137198:0:261288:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.137200:0:261288:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.137202:0:261288:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.137204:0:261288:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.137206:0:261288:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.137209:0:261288:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.137215:0:261288:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.137217:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c879b0 00000002:00010000:0.0:1788572470.137219:0:261288:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.137222:0:261288:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.137224:0:261288:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.137226:0:261288:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.137233:0:261288:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.137236:0:261288:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.137240:0:261288:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.137243:0:261288:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.137247:0:261288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 117 lru_type: 1 02000000:00000001:0.0:1788572470.137255:0:261288:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.137260:0:261288:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.137263:0:261288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 117 lru_type: 1 00000002:00000001:0.0:1788572470.137269:0:261288:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.137271:0:261288:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.137273:0:261288:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.137275:0:261288:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.137276:0:261288:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.137278:0:261288:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.137281:0:261288:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.137283:0:261288:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.137286:0:261288:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572470.137292:0:261288:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.137293:0:261288:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.137296:0:261288:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.137299:0:261288:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.137302:0:261288:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.137304:0:261288:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.137305:0:261288:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.137307:0:261288:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.137309:0:261288:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.137310:0:261288:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.137313:0:261288:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.137315:0:261288:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.137316:0:261288:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.137318:0:261288:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.137320:0:261288:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.137333:0:261288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c87bf8 00010000:00000001:0.0:1788572470.137336:0:261288:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.137337:0:261288:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.137338:0:261288:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.137340:0:261288:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.137342:0:261288:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.137344:0:261288:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.137347:0:261288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 117 lru_type: 1 00010000:00000001:0.0:1788572470.137352:0:261288:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.137353:0:261288:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.137355:0:261288:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.137356:0:261288:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.137358:0:261288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 118 lru_type: 1 00010000:00000001:0.0:1788572470.137363:0:261288:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.137365:0:261288:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.137366:0:261288:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.137368:0:261288:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.137369:0:261288:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.137371:0:261288:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.137372:0:261288:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.137373:0:261288:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.137375:0:261288:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.137377:0:261288:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.137379:0:261288:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.137381:0:261288:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.137382:0:261288:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.137383:0:261288:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.137385:0:261288:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.137387:0:261288:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.137390:0:261288:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.137394:0:261288:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.137396:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87b88 00000080:00002000:0.0:1788572470.137398:0:261288:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 9 00000080:00000001:0.0:1788572470.137400:0:261288:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.137402:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87bd0 00000080:00200000:0.0:1788572470.137403:0:261288:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.137408:0:261288:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.137411:0:261288:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.137413:0:261288:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.137415:0:261288:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.137417:0:261288:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.137418:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87be8 00000080:00002000:0.0:1788572470.137419:0:261288:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-009(ffff9c3708141000) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.137421:0:261288:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.137423:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87c30 00000080:00200000:0.0:1788572470.137424:0:261288:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-009, flags=9 00000080:00000001:0.0:1788572470.137434:0:261288:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.137436:0:261288:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.137438:0:261288:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.137440:0:261288:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c87e40 00000080:00002000:0.0:1788572470.137441:0:261288:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-009 (ffff9c3708141000, parent ffff9c3719d93af8, inode ffff9c372f0fec08) hashed, 00000080:00000001:0.0:1788572470.137444:0:261288:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.151435:0:261290:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.151442:0:261290:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.151445:0:261290:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.151448:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bb20 00000080:00200000:0.0:1788572470.151453:0:261290:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.151461:0:261290:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572470.151468:0:261290:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.151473:0:261290:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.151479:0:261290:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.151481:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0ba40 00800000:00000002:0.0:1788572470.151484:0:261290:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.151488:0:261290:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.151491:0:261290:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.151493:0:261290:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.151496:0:261290:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.151498:0:261290:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.151500:0:261290:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.151503:0:261290:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.151505:0:261290:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.151765:0:261290:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.151773:0:261290:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.151775:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b910 00000002:00010000:0.0:1788572470.151778:0:261290:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.151782:0:261290:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.151784:0:261290:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.151787:0:261290:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.151794:0:261290:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.151797:0:261290:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.151802:0:261290:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.151806:0:261290:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.151811:0:261290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 118 lru_type: 1 02000000:00000001:0.0:1788572470.151822:0:261290:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.151829:0:261290:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.151833:0:261290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 118 lru_type: 1 00000002:00000001:0.0:1788572470.151839:0:261290:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.151842:0:261290:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.151844:0:261290:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.151845:0:261290:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.151848:0:261290:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.151850:0:261290:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.151853:0:261290:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.151856:0:261290:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.151859:0:261290:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572470.151867:0:261290:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.151870:0:261290:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.151882:0:261290:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.151886:0:261290:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.151889:0:261290:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.151891:0:261290:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.151892:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.151894:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.151897:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.151900:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.151903:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.151905:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.151906:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.151908:0:261290:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.151909:0:261290:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.151912:0:261290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a0bb58 00010000:00000001:0.0:1788572470.151913:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.151914:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.151916:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.151918:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.151920:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.151922:0:261290:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.151925:0:261290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 118 lru_type: 1 00010000:00000001:0.0:1788572470.151931:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.151932:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.151934:0:261290:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.151936:0:261290:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.153178:0:261290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 119 lru_type: 1 00010000:00000001:0.0:1788572470.153195:0:261290:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.153198:0:261290:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.153200:0:261290:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.153205:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.153206:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.153208:0:261290:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153212:0:261290:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.153214:0:261290:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153216:0:261290:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153219:0:261290:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153221:0:261290:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.153222:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.153223:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.153227:0:261290:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.153229:0:261290:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.153232:0:261290:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.153238:0:261290:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.153243:0:261290:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.153246:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bae8 00000080:00002000:0.0:1788572470.153248:0:261290:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 9 00000080:00000001:0.0:1788572470.153250:0:261290:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.153252:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bb30 00000080:00200000:0.0:1788572470.153254:0:261290:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.153259:0:261290:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.153263:0:261290:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.153265:0:261290:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.153268:0:261290:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.153277:0:261290:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fa00. 00000020:00000001:0.0:1788572470.153280:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0bae8 00000080:00200000:0.0:1788572470.153282:0:261290:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-010, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.153286:0:261290:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.153287:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b998 00000080:00200000:0.0:1788572470.153289:0:261290:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-010, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.153296:0:261290:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572470.153300:0:261290:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.153304:0:261290:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.153307:0:261290:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.153308:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b8b8 00800000:00000002:0.0:1788572470.153311:0:261290:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-010' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.153314:0:261290:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.153317:0:261290:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.153320:0:261290:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.153656:0:261290:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.153659:0:261290:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.153661:0:261290:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.153663:0:261290:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.153666:0:261290:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.153670:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b850 00800000:00000002:0.0:1788572470.153672:0:261290:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-010' -> mds #0 00000002:00000001:0.0:1788572470.153677:0:261290:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.153679:0:261290:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a0b788 00000002:00010000:0.0:1788572470.153680:0:261290:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-010,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.153685:0:261290:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.153688:0:261290:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.153693:0:261290:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572470.153697:0:261290:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.153704:0:261290:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.153708:0:261290:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.153711:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.153713:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.153714:0:261290:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.153716:0:261290:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.153719:0:261290:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.153723:0:261290:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.153729:0:261290:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572470.153736:0:261290:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153741:0:261290:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.153742:0:261290:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153744:0:261290:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.153745:0:261290:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.153746:0:261290:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.153754:0:261290:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572470.153758:0:261290:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.153760:0:261290:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.153766:0:261290:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.153769:0:261290:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.153771:0:261290:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.153775:0:261290:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b000. 00000020:00000001:0.0:1788572470.153779:0:261290:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.153781:0:261290:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b000 with handle 0x1441586093168303 to hash 00000020:00000001:0.0:1788572470.153783:0:261290:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.153784:0:261290:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.153788:0:261290:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.153791:0:261290:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.153793:0:261290:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.153797:0:261290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.153805:0:261290:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.153816:0:261290:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261290 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.153821:0:261290:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.153822:0:261290:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.153829:0:261290:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.153831:0:261290:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572470.153834:0:261290:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.153835:0:261290:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572470.153840:0:261290:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.153844:0:261290:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.153846:0:261290:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.153847:0:261290:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.153859:0:261290:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.153867:0:261290:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.153870:0:261290:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.153891:0:261290:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.153893:0:261290:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.153895:0:261290:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.153899:0:261290:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261290:1875440809932032:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.153903:0:261290:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.153908:0:261290:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.153910:0:261290:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.153912:0:261290:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.153918:0:261290:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572470.153920:0:261290:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.153928:0:261290:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.153932:0:261290:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b00. 00000400:00000010:0.0:1788572470.153954:0:261290:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572470.153962:0:261290:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809932032, portal 10 00000100:00000001:0.0:1788572470.153964:0:261290:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.153965:0:261290:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572470.153971:0:261290:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.153979:0:261290:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.153981:0:261290:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.153984:0:261290:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572470.153987:0:261290:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809932032, offset 0 00000400:00000200:0.0:1788572470.154492:0:261290:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.154502:0:261290:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.154756:0:261290:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.154761:0:261290:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.154765:0:261290:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.154769:0:261290:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.154772:0:261290:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.154775:0:261290:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539661:539661:256:4294967295] 192.168.202.148@tcp LPNI seq info [539626:539626:8:4294967295] 00000400:00000200:0.0:1788572470.154780:0:261290:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.154789:0:261290:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.154797:0:261290:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.154804:0:261290:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572470.154812:0:261290:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.154820:0:261290:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.154825:0:261290:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.154905:0:261290:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.154911:0:261290:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14940 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.154915:0:261290:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.154918:0:261290:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609176 00000100:00000001:0.0:1788572470.154922:0:261290:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609319 00000100:00000001:0.0:1788572470.154925:0:261290:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609176 00000100:00000001:0.0:1788572470.154926:0:261290:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609319 00000100:00000001:0.0:1788572470.154929:0:261290:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.154932:0:261290:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.154934:0:261290:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.154946:0:261290:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.154949:0:261290:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.154952:0:261290:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.154954:0:261290:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.163259:0:261290:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.163337:0:261290:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.163339:0:261290:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.163343:0:261290:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8b00. 00000400:00000200:0.0:1788572470.163348:0:261290:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572470.163350:0:261290:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572470.163353:0:261290:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.163358:0:261290:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.163367:0:261290:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.163373:0:261290:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.163375:0:261290:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.163378:0:261290:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.163381:0:261290:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.163383:0:261290:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.163385:0:261290:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.163388:0:261290:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.163397:0:261290:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=34 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572470.163402:0:261290:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.163405:0:261290:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.163407:0:261290:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.163410:0:261290:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.163413:0:261290:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.163417:0:261290:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.163419:0:261290:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.163421:0:261290:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.163424:0:261290:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.163429:0:261290:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.163436:0:261290:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.163438:0:261290:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.163440:0:261290:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.163444:0:261290:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.163451:0:261290:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261290:1875440809932032:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.163456:0:261290:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.163459:0:261290:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.163461:0:261290:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572470.163463:0:261290:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.163467:0:261290:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.163473:0:261290:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572470.163476:0:261290:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.163479:0:261290:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572470.163481:0:261290:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.163494:0:261290:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.163498:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.163500:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.163503:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=4 00000020:00000001:0.0:1788572470.163506:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.163509:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000040:0.0:1788572470.163516:0:261290:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6b000, remote cookie: 0xd9894672d99a1e07, flags: 0x1001 00010000:00010000:0.0:1788572470.163521:0:261290:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.163530:0:261290:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xd9e:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572470.163533:0:261290:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572470.163540:0:261290:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7e00 count: 3 00010000:00000001:0.0:1788572470.163543:0:261290:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.163545:0:261290:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.163548:0:261290:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.163565:0:261290:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.163572:0:261290:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.163576:0:261290:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572470.163578:0:261290:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572470.163581:0:261290:0:(ldlm_lock.c:1024:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572470.163582:0:261290:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572470.163584:0:261290:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xd9e:0x0].0x0 (ffff9c37050f7e00) refcount = 3 00010000:00000040:0.0:1788572470.163586:0:261290:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.163589:0:261290:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee200/0x144158609316829a lrc: 1/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ce8 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000040:0.0:1788572470.163602:0:261290:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb000/0x1441586093167d53 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a0a7a expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572470.163609:0:261290:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.163615:0:261290:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572470.163618:0:261290:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.163620:0:261290:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.163622:0:261290:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.163625:0:261290:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.163628:0:261290:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.164049:0:261290:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.164052:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164054:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.164055:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164056:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.164058:0:261290:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.164063:0:261290:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572470.164064:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.164066:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.164068:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=3 00000020:00000001:0.0:1788572470.164070:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164072:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164075:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.164076:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.164077:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=4 00000020:00000001:0.0:1788572470.164078:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164080:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164083:0:261290:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.164087:0:261290:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.164090:0:261290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.164097:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164098:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.164099:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.164101:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.164102:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=5 00000020:00000001:0.0:1788572470.164104:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164106:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164108:0:261290:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.164111:0:261290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.164117:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164118:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.164121:0:261290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.164126:0:261290:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.164128:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164129:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.164130:0:261290:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.164131:0:261290:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572470.164137:0:261290:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b6d80 x1875440809932032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572470.164146:0:261290:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.164147:0:261290:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.164148:0:261290:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=3 00000020:00000001:0.0:1788572470.164150:0:261290:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.164152:0:261290:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00000002:00010000:0.0:1788572470.164155:0:261290:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.165908:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.165910:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.165914:0:237084:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.165917:0:237084:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.165919:0:237084:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.165924:0:237084:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.165931:0:237084:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372eb6b000 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.165934:0:237084:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.165938:0:237084:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.165945:0:237084:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.165947:0:237084:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.165950:0:237084:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.165956:0:237084:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.165958:0:237084:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.165959:0:237084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.165961:0:237084:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.165962:0:237084:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=3 00000020:00000001:0.0:1788572470.165964:0:237084:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.165966:0:237084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.165969:0:237084:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.165972:0:237084:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.165982:0:237084:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.165985:0:237084:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.165987:0:237084:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.165989:0:237084:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xd9e:0x0] mode (null) 00000002:00000001:0.0:1788572470.165991:0:237084:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.165993:0:237084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.165998:0:237084:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7e00 count: 4 00010000:00000001:0.0:1788572470.166000:0:237084:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.166003:0:237084:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.166005:0:237084:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.166006:0:237084:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.166009:0:237084:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.166012:0:237084:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ee200/0x144158609316829a lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ce8 expref: -99 pid: 261278 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00000001:0.0:1788572470.166018:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.166019:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.166021:0:237084:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.166022:0:237084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.166023:0:237084:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.166024:0:237084:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ee200 refcount=2 00000020:00000001:0.0:1788572470.166026:0:237084:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609811456 : -109714099740160 : ffff9c37320ee200) 00010000:00000001:0.0:1788572470.166028:0:237084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609811456 : -109714099740160 : ffff9c37320ee200) 00010000:00000001:0.0:1788572470.166030:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.166031:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.166032:0:237084:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.166034:0:237084:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.166035:0:237084:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.166038:0:237084:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.166040:0:237084:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.166043:0:237084:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.166044:0:237084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.166046:0:237084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.166048:0:237084:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372eb6b000 with handle 0x1441586093168303 from hash 00010000:00000001:0.0:1788572470.166051:0:237084:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.166052:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.166053:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.166054:0:237084:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.166057:0:237084:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.166058:0:237084:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.166062:0:237084:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.166064:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.166065:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.166067:0:237084:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.166069:0:237084:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.166071:0:237084:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.166076:0:237084:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572470.166078:0:237084:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.166081:0:237084:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.166083:0:237084:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.166085:0:237084:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.166089:0:237084:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572470.166093:0:237084:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.166096:0:237084:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.166100:0:237084:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.166106:0:237084:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.166108:0:237084:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.166111:0:237084:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.166113:0:237084:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.166119:0:237084:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.166140:0:237084:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.166142:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.166144:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.166145:0:237084:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.166147:0:237084:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.166149:0:237084:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.166152:0:237084:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.166158:0:237084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.166160:0:237084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168303 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xd9e:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a1e07 expref: -99 pid: 261290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.166167:0:237084:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83200. 00010000:00000001:0.0:1788572470.166175:0:237084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.166176:0:237084:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.166178:0:237084:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c372d946000. 00010000:00000001:0.0:1788572470.166181:0:237084:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.166183:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.166184:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.166186:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.166188:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.166189:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.166190:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.166192:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.166193:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.166194:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572470.166201:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.166203:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.166204:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.166209:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.166217:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.166218:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.166220:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.166222:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.166224:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.166227:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809932288:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.166230:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.166233:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.166235:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.166236:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.166239:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b6400. 02000000:00000001:0.0:1788572470.166242:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.166246:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.166249:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b00. 00000400:00000010:0.0:1788572470.166253:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572470.166256:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809932288, portal 18 00000100:00000001:0.0:1788572470.166258:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.166259:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572470.166265:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.166271:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.166273:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.166275:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572470.166278:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809932288, offset 0 00000400:00000200:0.0:1788572470.166282:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.166286:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.166291:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.166294:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.166296:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.166298:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.166300:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.166303:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539663:539663:256:4294967295] 192.168.202.148@tcp LPNI seq info [539628:539628:8:4294967295] 00000400:00000200:0.0:1788572470.166307:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.166313:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.166318:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.166322:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.166326:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.166330:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.166333:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.166344:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.166347:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14938 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.166350:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.166352:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609187 00000100:00000001:0.0:1788572470.166355:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609344 00000100:00000001:0.0:1788572470.166357:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609187 00000100:00000001:0.0:1788572470.166359:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609344 00000100:00000001:0.0:1788572470.166360:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.166362:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.166365:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.166366:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.166370:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.166371:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572470.166373:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.166374:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.166376:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.166377:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.166396:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.166399:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572470.166403:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.166407:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.166410:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788572470.166412:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788572470.166415:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.166419:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.166427:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.166430:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.166438:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.166439:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.172402:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.172407:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.172414:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.172416:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.172419:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2058. 00000400:00000200:0.0:1788572470.172423:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572470.172425:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572470.172428:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.172432:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440809932160/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.172440:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440809932160/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.172446:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.172447:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.172450:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.172453:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.172455:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.172457:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.172459:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.172468:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=34 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.172472:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=85 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572470.172475:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.172476:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.172479:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.172494:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.172498:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.172500:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572470.172502:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.172505:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.172509:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440809932160/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.172516:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.172519:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.172521:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572470.172524:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572470.172526:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172529:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172532:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=4 00000020:00000001:0.0:1788572470.172534:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172537:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172539:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172540:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172541:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=5 00000020:00000001:0.0:1788572470.172543:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172545:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172548:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.172552:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.172556:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172567:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.172569:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.172571:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.172573:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172574:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172576:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=6 00000020:00000001:0.0:1788572470.172577:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172579:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000040:0.0:1788572470.172585:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204bc00, remote cookie: 0xd9894672d99a1e0e, flags: 0x1001 00010000:00010000:0.0:1788572470.172590:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172597:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572470.172600:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.172602:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.172606:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572470.172611:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.172626:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x5260:0x0].0x0 (ffff9c37050f7a00) refcount = 1 00010000:00000040:0.0:1788572470.172628:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.172631:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172671:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.172673:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.172676:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.172678:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.172682:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172688:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.172690:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.172691:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.172692:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.172693:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.172694:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572470.172697:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572470.172699:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172701:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172703:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=5 00000020:00000001:0.0:1788572470.172705:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172707:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00000008:00000001:0.0:1788572470.172711:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572470.172715:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000008:00010000:0.0:1788572470.172719:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.172726:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572470.172729:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572470.172731:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572470.172734:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572470.172737:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.172739:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.172740:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000008:00000001:0.0:1788572470.172742:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572470.172744:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.172746:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572470.172762:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572470.172763:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.172765:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172766:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172768:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=6 00000020:00000001:0.0:1788572470.172770:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172772:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172774:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.172777:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172783:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.172784:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.172787:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.172794:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.172795:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.172796:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.172797:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.172799:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.172800:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.172801:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204bc00 refcount=5 00000020:00000001:0.0:1788572470.172803:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.172805:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609146368 : -109714100405248 : ffff9c373204bc00) 00010000:00000001:0.0:1788572470.173001:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.173004:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.173011:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.173012:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.173013:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.173016:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.173018:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572470.173025:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.173028:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.173030:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.173033:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.173034:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.173036:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.173039:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.173040:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.173042:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.173045:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.173047:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.173048:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.173049:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.173050:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.173051:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.173052:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.173058:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440809932160/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.173078:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809932160:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.173085:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.173088:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.173091:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440809932160/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.173099:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.173101:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.173105:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c3718ec3000. 02000000:00000001:0.0:1788572470.173109:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.173112:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.173114:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572470.173116:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.173119:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215e00. 02000000:00000001:0.0:1788572470.173124:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.173126:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.173128:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788572470.173131:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.173132:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.173134:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.173141:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.173142:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.173145:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.173147:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.181237:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.181255:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.181260:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.181263:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.181271:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.181279:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.181286:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.181291:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.181297:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.181301:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41745f600 00000400:00000200:0.0:1788572470.181307:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.181313:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529909 [1] + 224 00000800:00000001:0.0:1788572470.181320:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.182717:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.182726:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.182735:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.182743:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.182885:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.182896:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5c00 x1875440809932288/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.183846:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.183862:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.183870:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.195602:0:261292:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.195609:0:261292:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.195612:0:261292:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.195615:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97b20 00000080:00200000:0.0:1788572470.195620:0:261292:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.195627:0:261292:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572470.195652:0:261292:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.195657:0:261292:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.195662:0:261292:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.195664:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97a40 00800000:00000002:0.0:1788572470.195667:0:261292:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.195671:0:261292:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.195674:0:261292:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.195676:0:261292:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.195678:0:261292:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.195681:0:261292:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.195683:0:261292:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.195685:0:261292:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.195687:0:261292:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.195690:0:261292:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.195696:0:261292:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.195697:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97910 00000002:00010000:0.0:1788572470.195699:0:261292:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.195703:0:261292:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.195705:0:261292:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.195708:0:261292:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.195716:0:261292:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.195718:0:261292:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.195723:0:261292:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.195727:0:261292:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.195732:0:261292:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 120 lru_type: 1 02000000:00000001:0.0:1788572470.195743:0:261292:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.195749:0:261292:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.195752:0:261292:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 120 lru_type: 1 00000002:00000001:0.0:1788572470.195759:0:261292:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.195761:0:261292:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.195763:0:261292:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.195765:0:261292:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.195766:0:261292:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.195768:0:261292:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.195771:0:261292:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.195774:0:261292:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.195777:0:261292:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6000. 00000080:00000001:0.0:1788572470.195785:0:261292:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.195786:0:261292:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.195789:0:261292:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.195793:0:261292:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.195796:0:261292:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.195799:0:261292:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.195800:0:261292:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.195803:0:261292:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.195806:0:261292:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.196564:0:261292:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.196571:0:261292:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.196574:0:261292:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.196576:0:261292:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.196578:0:261292:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.196580:0:261292:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.196584:0:261292:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c97b58 00010000:00000001:0.0:1788572470.196586:0:261292:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.196587:0:261292:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.196589:0:261292:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.196591:0:261292:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.196593:0:261292:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.196595:0:261292:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.196599:0:261292:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 120 lru_type: 1 00010000:00000001:0.0:1788572470.196607:0:261292:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.196608:0:261292:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.196609:0:261292:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.196611:0:261292:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.196614:0:261292:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 121 lru_type: 1 00010000:00000001:0.0:1788572470.196621:0:261292:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.196624:0:261292:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.196627:0:261292:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.196631:0:261292:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.196632:0:261292:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.196633:0:261292:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.196656:0:261292:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.196658:0:261292:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.196661:0:261292:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.196664:0:261292:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.196666:0:261292:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.196668:0:261292:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.196669:0:261292:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.196672:0:261292:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.196674:0:261292:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.196676:0:261292:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.196682:0:261292:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.196687:0:261292:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.196689:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97ae8 00000080:00002000:0.0:1788572470.196691:0:261292:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 10 00000080:00000001:0.0:1788572470.196694:0:261292:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.196696:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97b30 00000080:00200000:0.0:1788572470.196698:0:261292:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.196704:0:261292:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.196708:0:261292:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.196710:0:261292:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.196713:0:261292:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.196723:0:261292:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f7c0. 00000020:00000001:0.0:1788572470.196726:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97ae8 00000080:00200000:0.0:1788572470.196728:0:261292:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-011, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.196731:0:261292:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.196733:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97998 00000080:00200000:0.0:1788572470.196735:0:261292:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-011, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.196742:0:261292:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572470.196746:0:261292:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.196750:0:261292:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.196752:0:261292:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.196754:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c978b8 00800000:00000002:0.0:1788572470.196755:0:261292:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-011' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.196759:0:261292:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.196762:0:261292:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.196764:0:261292:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.196766:0:261292:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.196767:0:261292:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.196769:0:261292:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.196770:0:261292:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.196772:0:261292:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.196775:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97850 00800000:00000002:0.0:1788572470.196777:0:261292:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-011' -> mds #0 00000002:00000001:0.0:1788572470.196781:0:261292:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.196783:0:261292:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c97788 00000002:00010000:0.0:1788572470.196784:0:261292:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-011,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.196788:0:261292:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.196791:0:261292:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.196796:0:261292:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572470.196800:0:261292:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.196806:0:261292:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.197570:0:261292:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.197574:0:261292:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.197576:0:261292:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.197577:0:261292:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.197581:0:261292:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.197583:0:261292:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.197587:0:261292:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.197597:0:261292:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572470.197606:0:261292:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.197611:0:261292:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.197613:0:261292:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.197614:0:261292:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.197615:0:261292:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.197617:0:261292:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.197625:0:261292:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00010000:00000001:0.0:1788572470.197629:0:261292:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.197631:0:261292:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.197663:0:261292:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.197667:0:261292:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.197669:0:261292:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.197674:0:261292:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b000. 00000020:00000001:0.0:1788572470.197678:0:261292:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.197680:0:261292:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b000 with handle 0x1441586093168311 to hash 00000020:00000001:0.0:1788572470.197682:0:261292:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.197683:0:261292:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.197687:0:261292:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.197690:0:261292:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.197693:0:261292:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.197697:0:261292:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168311 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261292 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.197706:0:261292:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168311 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261292 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.197715:0:261292:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168311 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261292 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.197721:0:261292:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.197723:0:261292:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.197730:0:261292:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.197732:0:261292:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572470.197734:0:261292:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.197735:0:261292:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000001:0.0:1788572470.197740:0:261292:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.197744:0:261292:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.197746:0:261292:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.197748:0:261292:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.197760:0:261292:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.197768:0:261292:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.197771:0:261292:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.197775:0:261292:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.197777:0:261292:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.197780:0:261292:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.197783:0:261292:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261292:1875440809932416:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.197787:0:261292:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.197793:0:261292:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.197796:0:261292:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.197799:0:261292:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.197803:0:261292:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb000. 02000000:00000001:0.0:1788572470.197805:0:261292:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.198000:0:261292:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.198006:0:261292:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8630. 00000400:00000010:0.0:1788572470.198014:0:261292:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572470.198020:0:261292:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809932416, portal 10 00000100:00000001:0.0:1788572470.198022:0:261292:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.198023:0:261292:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000040:0.0:1788572470.198030:0:261292:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.198037:0:261292:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.198040:0:261292:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.198043:0:261292:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572470.198046:0:261292:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809932416, offset 0 00000400:00000200:0.0:1788572470.198052:0:261292:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.198060:0:261292:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.198998:0:261292:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.199005:0:261292:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.199010:0:261292:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.199014:0:261292:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.199017:0:261292:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.199021:0:261292:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539664:539664:256:4294967295] 192.168.202.148@tcp LPNI seq info [539629:539629:8:4294967295] 00000400:00000200:0.0:1788572470.199026:0:261292:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.199036:0:261292:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.199043:0:261292:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.199051:0:261292:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.199058:0:261292:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.199139:0:261292:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.199144:0:261292:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.200162:0:261292:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.200171:0:261292:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14942 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.200180:0:261292:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.200184:0:261292:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609220 00000100:00000001:0.0:1788572470.200187:0:261292:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609344 00000100:00000001:0.0:1788572470.200190:0:261292:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609220 00000100:00000001:0.0:1788572470.200193:0:261292:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609344 00000100:00000001:0.0:1788572470.200195:0:261292:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.200199:0:261292:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.200202:0:261292:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.200204:0:261292:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.200207:0:261292:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.200210:0:261292:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.200213:0:261292:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.208191:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.208201:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572470.208208:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.208217:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.208224:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31990 00000400:00000010:0.0:1788572470.208227:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31990. 00000100:00000001:0.0:1788572470.208232:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.208240:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.208254:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.208257:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.208265:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.208267:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.208541:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.208609:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.208612:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.208617:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.208624:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.208629:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.208901:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.208906:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.208911:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.208916:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b41745f680 00000400:00000200:0.0:1788572470.208921:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.208927:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529919 [1] + 224 00000800:00000001:0.0:1788572470.208935:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.208943:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.208946:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.208950:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.208955:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.208961:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.208969:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b6300 x1875440809932416/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.208994:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.209001:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.209004:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.210840:0:261292:0:(statahead.c:1163:ll_start_agl()) Process entered 00000080:00400000:0.0:1788572470.210843:0:261292:0:(statahead.c:1166:ll_start_agl()) start agl thread: sai ffff9c3711215c00, parent d434.sanity 00000080:00000001:0.0:1788572470.213216:0:261295:0:(statahead.c:1097:ll_agl_thread()) Process entered 00000080:00400000:0.0:1788572470.213219:0:261295:0:(statahead.c:1100:ll_agl_thread()) agl thread started: sai ffff9c3711215c00, parent d434.sanity 00000080:00000001:0.0:1788572470.213223:0:261295:0:(statahead.c:681:ll_agl_trigger()) Process entered 00000080:00000001:0.0:1788572470.213225:0:261295:0:(statahead.c:733:ll_agl_trigger()) Process leaving 00000080:00000001:0.0:1788572470.213226:0:261295:0:(statahead.c:681:ll_agl_trigger()) Process entered 00000080:00000001:0.0:1788572470.213228:0:261295:0:(statahead.c:711:ll_agl_trigger()) Process leaving 00000100:00000001:0.0:1788572470.213422:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.213427:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.213430:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.213436:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.213443:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.213445:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.213449:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.213451:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.213454:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.213457:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809932544:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.213463:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.213469:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.213471:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.213473:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.213479:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572470.213481:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.213489:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.213493:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c82c0. 00000400:00000010:0.0:1788572470.213501:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572470.213508:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809932544, portal 4 00000100:00000001:0.0:1788572470.213510:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.213511:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788572470.213517:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.213523:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.213525:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.213527:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572470.213530:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809932544, offset 0 00000400:00000200:0.0:1788572470.213537:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.213545:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.213552:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.213557:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.213561:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.213565:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.213567:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.213572:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539665:539665:256:4294967295] 192.168.202.148@tcp LPNI seq info [539630:539630:8:4294967295] 00000400:00000200:0.0:1788572470.213576:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.213585:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.213591:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.213596:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.213602:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.213610:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.213616:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.213627:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.213631:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 182670 usecs (183 jiffies) 00000400:00000200:0.0:1788572470.213652:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.213655:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609402 00000100:00000001:0.0:1788572470.213663:0:2404:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307609344 00000100:00000001:0.0:1788572470.213668:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609402 00000100:00000001:0.0:1788572470.213672:0:2404:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307609344 00000100:00000001:0.0:1788572470.213674:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.213677:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.213680:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.213682:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.223023:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.223026:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572470.223030:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.223033:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.227672:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.227678:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.227683:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c82c0. 00000400:00000200:0.0:1788572470.227689:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31990 00000400:00000010:0.0:1788572470.227692:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31990. 00000100:00000001:0.0:1788572470.227697:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.227706:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.227717:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.227725:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.227727:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.227730:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.227733:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.227736:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.227740:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.227743:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.227752:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=34 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.227756:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=7 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572470.227759:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.227760:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.227763:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.227767:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.227770:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.227772:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572470.227774:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.227777:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.227782:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.227789:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.227793:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.227794:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572470.227797:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572470.227800:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.227802:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.227806:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=4 00000020:00000001:0.0:1788572470.227883:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.227886:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.227889:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.227890:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.227891:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=5 00000020:00000001:0.0:1788572470.227893:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.227895:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.227898:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.227901:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.227904:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.227918:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.227920:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.227921:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.227924:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.227925:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.227926:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=6 00000020:00000001:0.0:1788572470.227927:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.227929:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000040:0.0:1788572470.227936:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204a200, remote cookie: 0xd9894672d99a1e7e, flags: 0x1001 00010000:00010000:0.0:1788572470.227939:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.227946:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572470.227949:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.227951:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.227955:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572470.227960:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.227969:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51ee:0x0].0x0 (ffff9c37050f7d00) refcount = 1 00010000:00000040:0.0:1788572470.227971:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.227974:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.227983:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.227985:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.227987:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.227990:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.227994:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.228001:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.228002:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228003:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.228004:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228005:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.228007:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572470.228009:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572470.228012:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.228013:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.228015:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=5 00000020:00000001:0.0:1788572470.228016:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.228018:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00000008:00000001:0.0:1788572470.228022:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572470.228026:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9340 -> ffff9c37313e1210 00000008:00010000:0.0:1788572470.228030:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.228038:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9340 -> ffff9c37313e1210 00000020:00000001:0.0:1788572470.228040:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572470.228042:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572470.228045:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572470.228048:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.228050:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.228052:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9340 -> ffff9c37313e1210 00000008:00000001:0.0:1788572470.228053:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572470.228055:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.228057:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572470.228063:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572470.228064:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228137:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.228139:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.228141:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=6 00000020:00000001:0.0:1788572470.228143:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.228146:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.228148:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.228151:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.228158:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228159:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.228162:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.228169:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.228170:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228171:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.228172:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228174:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.228175:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.228177:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a200 refcount=5 00000020:00000001:0.0:1788572470.228179:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.228180:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609139712 : -109714100411904 : ffff9c373204a200) 00010000:00000001:0.0:1788572470.228182:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.228185:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.228191:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228191:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.228193:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.228195:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.228198:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000d93ead8c/0x1441586093168342 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a1e7e expref: -99 pid: 261292 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572470.228205:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.228207:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.228209:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.228213:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228214:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.228216:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228218:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.228220:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228223:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228225:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.228227:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.228228:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228229:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.228230:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.228231:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.228232:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.228237:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.228246:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809932544:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.228251:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.228254:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.228258:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440809932544/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.228264:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.228267:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.228270:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572470.228274:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.228276:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.228278:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572470.228280:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.228283:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215800. 02000000:00000001:0.0:1788572470.228287:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.228289:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.228291:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788572470.228294:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.228295:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.228298:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.228304:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.228306:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.228308:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.228310:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.265626:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.265632:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.265675:0:261296:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.265679:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7b20 00000080:00200000:0.0:1788572470.265683:0:261296:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.265691:0:261296:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572470.265698:0:261296:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.265701:0:261296:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.265705:0:261296:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.265707:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7a40 00800000:00000002:0.0:1788572470.265710:0:261296:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.265713:0:261296:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.265716:0:261296:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.265718:0:261296:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.265720:0:261296:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.265723:0:261296:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.265725:0:261296:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.265727:0:261296:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.265729:0:261296:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.265732:0:261296:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.265736:0:261296:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.265738:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7910 00000002:00010000:0.0:1788572470.265739:0:261296:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.265743:0:261296:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.265745:0:261296:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.265748:0:261296:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.265754:0:261296:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.265757:0:261296:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.265761:0:261296:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.265765:0:261296:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.265769:0:261296:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 02000000:00000001:0.0:1788572470.265779:0:261296:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.265783:0:261296:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.265786:0:261296:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 00000002:00000001:0.0:1788572470.265793:0:261296:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.265795:0:261296:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.265797:0:261296:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.265798:0:261296:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.265800:0:261296:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.265803:0:261296:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.265806:0:261296:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.265808:0:261296:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.265812:0:261296:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572470.265819:0:261296:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.265821:0:261296:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.265824:0:261296:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.265827:0:261296:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.265830:0:261296:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.265832:0:261296:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.265834:0:261296:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.265836:0:261296:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.265838:0:261296:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.265840:0:261296:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.265842:0:261296:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.265844:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.265846:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.265847:0:261296:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.265849:0:261296:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.265851:0:261296:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1ca7b58 00010000:00000001:0.0:1788572470.265853:0:261296:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.265854:0:261296:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.265855:0:261296:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.265857:0:261296:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.265859:0:261296:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.265862:0:261296:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.265864:0:261296:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 00010000:00000001:0.0:1788572470.265870:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.265881:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.265883:0:261296:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.265885:0:261296:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.265888:0:261296:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00010000:00000001:0.0:1788572470.265895:0:261296:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.265898:0:261296:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.265899:0:261296:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.265903:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.265904:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.265905:0:261296:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.265908:0:261296:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.265910:0:261296:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.265912:0:261296:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.265915:0:261296:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.265917:0:261296:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.265919:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.265920:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.265923:0:261296:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.265925:0:261296:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.265927:0:261296:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.265932:0:261296:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.265935:0:261296:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.265938:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7ae8 00000080:00002000:0.0:1788572470.265939:0:261296:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 12 00000080:00000001:0.0:1788572470.265942:0:261296:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.265944:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7b30 00000080:00200000:0.0:1788572470.265946:0:261296:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.265951:0:261296:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.265955:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.265956:0:261296:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.265959:0:261296:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.265967:0:261296:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f480. 00000020:00000001:0.0:1788572470.265970:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7ae8 00000080:00200000:0.0:1788572470.265973:0:261296:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-012, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.265976:0:261296:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.265977:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7998 00000080:00200000:0.0:1788572470.265979:0:261296:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-012, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000001:0.0:1788572470.265984:0:261296:0:(statahead.c:1997:revalidate_statahead_dentry()) Process entered 00000080:00000020:0.0:1788572470.265988:0:261296:0:(statahead.c:273:sa_get()) lustre: wrong pid=261292:261296 for entry f434.sanity-012 00000080:00000001:0.0:1788572470.266006:0:261296:0:(statahead.c:2046:revalidate_statahead_dentry()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00000080:00400000:0.0:1788572470.266009:0:261296:0:(statahead.c:2622:ll_revalidate_statahead()) revalidate statahead f434.sanity-012: rc = -22. 00000080:00000010:0.0:1788572470.266017:0:261296:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572470.266020:0:261296:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 16 00000080:00000001:0.0:1788572470.266023:0:261296:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.266025:0:261296:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.266027:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca78b8 00800000:00000002:0.0:1788572470.266029:0:261296:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-012' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.266032:0:261296:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.266034:0:261296:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.266036:0:261296:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.266037:0:261296:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.266039:0:261296:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.266040:0:261296:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.266042:0:261296:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.266044:0:261296:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.266047:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7850 00800000:00000002:0.0:1788572470.266048:0:261296:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-012' -> mds #0 00000002:00000001:0.0:1788572470.266051:0:261296:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.266053:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7788 00000002:00010000:0.0:1788572470.266054:0:261296:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-012,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.266059:0:261296:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.266061:0:261296:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.266065:0:261296:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572470.266070:0:261296:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=45 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.266075:0:261296:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.266079:0:261296:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.266081:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.266082:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.266084:0:261296:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.266087:0:261296:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.266089:0:261296:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.266091:0:261296:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.266097:0:261296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572470.266104:0:261296:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.266109:0:261296:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.266110:0:261296:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.266112:0:261296:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.266113:0:261296:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.266115:0:261296:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.266121:0:261296:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00010000:00000001:0.0:1788572470.266124:0:261296:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.266127:0:261296:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.266130:0:261296:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 15 00010000:00000001:0.0:1788572470.266133:0:261296:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.266135:0:261296:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.266138:0:261296:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6be00. 00000020:00000001:0.0:1788572470.266142:0:261296:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.266144:0:261296:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6be00 with handle 0x1441586093168388 to hash 00000020:00000001:0.0:1788572470.266146:0:261296:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.266148:0:261296:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572470.266151:0:261296:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572470.266154:0:261296:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.266156:0:261296:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.266158:0:261296:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168388 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 16 type: IBT flags: 0x10000000000000 pid: 261296 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.266166:0:261296:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168388 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 16 type: IBT flags: 0x0 pid: 261296 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.266175:0:261296:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168388 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 16 type: IBT flags: 0x0 pid: 261296 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.266180:0:261296:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.266182:0:261296:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.266188:0:261296:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5cc0. 00000100:00000001:0.0:1788572470.266190:0:261296:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716480 : -109714420835136 : ffff9c371eeb5cc0) 00000100:00000001:0.0:1788572470.266192:0:261296:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.266193:0:261296:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000001:0.0:1788572470.266198:0:261296:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.266202:0:261296:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.266204:0:261296:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.266206:0:261296:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.266216:0:261296:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.266223:0:261296:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.266225:0:261296:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.266229:0:261296:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.266232:0:261296:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.266234:0:261296:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.266237:0:261296:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261296:1875440809933056:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.266241:0:261296:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.266247:0:261296:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.266249:0:261296:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.266251:0:261296:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.266256:0:261296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ee000. 02000000:00000001:0.0:1788572470.266258:0:261296:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.266267:0:261296:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.266271:0:261296:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c82c0. 00000400:00000010:0.0:1788572470.266278:0:261296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572470.266286:0:261296:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809933056, portal 10 00000100:00000001:0.0:1788572470.266288:0:261296:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.266289:0:261296:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572470.266295:0:261296:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.266302:0:261296:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.266304:0:261296:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.266306:0:261296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572470.266309:0:261296:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809933056, offset 0 00000400:00000200:0.0:1788572470.266315:0:261296:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.266323:0:261296:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.266330:0:261296:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.266335:0:261296:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.266340:0:261296:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.266344:0:261296:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.266346:0:261296:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.266350:0:261296:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539669:539669:256:4294967295] 192.168.202.148@tcp LPNI seq info [539634:539634:8:4294967295] 00000400:00000200:0.0:1788572470.266355:0:261296:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.266364:0:261296:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.266369:0:261296:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.266375:0:261296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.266382:0:261296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.266389:0:261296:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.266394:0:261296:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.266409:0:261296:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.266414:0:261296:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14963 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.266417:0:261296:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.266420:0:261296:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609286 00000800:00000200:0.0:1788572470.266518:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.266523:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572470.266527:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.266533:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.266539:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31550 00000400:00000010:0.0:1788572470.266541:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31550. 00000100:00000001:0.0:1788572470.266546:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.266550:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.266559:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.266562:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.266568:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.266570:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.266586:0:261296:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609402 00000100:00000001:0.0:1788572470.266589:0:261296:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609286 00000100:00000001:0.0:1788572470.266591:0:261296:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609402 00000100:00000001:0.0:1788572470.266593:0:261296:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.266596:0:261296:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.266599:0:261296:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.266600:0:261296:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.266603:0:261296:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5cc0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.266606:0:261296:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.266608:0:261296:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.284420:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.284430:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.284448:0:261295:0:(statahead.c:1120:ll_agl_thread()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.284537:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296069:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.296073:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296079:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.296087:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296094:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.296103:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296108:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296113:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296117:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 680 into portal 4 MB=0x6a9b41745f980 00000400:00000200:0.0:1788572470.296122:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296129:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 680/680 into md 0x529961 [2] + 224 00000800:00000001:0.0:1788572470.296137:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296141:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.296145:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.296149:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.296157:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.296165:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5c00 x1875440809933184/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572555 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.296174:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=680 offset=224 replen=4320 req@ffff9c371f1c5c00 x1875440809933184/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 672/4320 e 0 to 0 dl 1788572555 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.296200:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.296208:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296212:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.296214:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296216:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.296219:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296223:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.296227:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296229:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296231:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296234:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b41745f900 00000400:00000200:0.0:1788572470.296238:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.296242:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529951 [1] + 224 00000800:00000001:0.0:1788572470.296246:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296276:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.296278:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.296282:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.296285:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.296290:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.296297:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4000 x1875440809933056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.296415:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.296422:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.296425:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.300466:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.300474:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788572470.300478:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.300483:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.300488:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1198 00000400:00000010:0.0:1788572470.300491:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1198. 00000100:00000001:0.0:1788572470.300494:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.300498:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68a80 x1875440809933312/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.300507:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.300509:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440809933312/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.300515:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.300516:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.306117:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.306135:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.306139:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.306142:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.306151:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.306158:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.306167:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.306172:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.306178:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.306182:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 4264 into portal 4 MB=0x6a9b41745f880 00000400:00000200:0.0:1788572470.306186:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.306193:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 4264/4264 into md 0x529949 [4] + 224 00000800:00000001:0.0:1788572470.306200:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.306212:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.306215:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.306219:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.306227:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.306238:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6bb80 x1875440809932928/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3872/8944 e 0 to 0 dl 1788572555 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.306249:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=4264 offset=224 replen=8944 req@ffff9c371ed6bb80 x1875440809932928/t0(0) o63->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:24/4 lens 3872/8944 e 0 to 0 dl 1788572555 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.306271:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.306280:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.306283:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.313954:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.313960:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5000. 00000400:00000200:0.0:1788572470.313967:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.313974:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.313981:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1550 00000400:00000010:0.0:1788572470.313983:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1550. 00000100:00000001:0.0:1788572470.313988:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.313996:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809933440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.314007:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.314011:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809933440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.314018:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.314020:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.314934:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.314948:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.314950:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.314953:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.314960:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.314966:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.314974:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.314978:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.314983:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.314987:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41745fa00 00000400:00000200:0.0:1788572470.314992:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.314998:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529971 [1] + 224 00000800:00000001:0.0:1788572470.315005:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.315011:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.315014:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.315017:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.315023:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.315027:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68a80 x1875440809933312/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.315036:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68a80 x1875440809933312/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.315057:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.315064:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.315443:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.321297:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.321312:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.321314:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.321317:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.321324:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.321330:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.321338:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.321343:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.321348:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.321352:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41745fa80 00000400:00000200:0.0:1788572470.321356:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.321362:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529979 [1] + 224 00000800:00000001:0.0:1788572470.321369:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.322124:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.322130:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.322137:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.322146:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.322154:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809933440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.322165:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809933440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.322185:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.322194:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.322198:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.322974:0:261296:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.322978:0:261296:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.322984:0:261296:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.322987:0:261296:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1148 -> ffff9c37313e1148 00000020:00000001:0.0:1788572470.322989:0:261296:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1148 -> ffff9c37313e1148 00000020:00000001:0.0:1788572470.322990:0:261296:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.322996:0:261296:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.323001:0:261296:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.323002:0:261296:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.323004:0:261296:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.323005:0:261296:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.323007:0:261296:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xda0:11031 on OST[0]: s=0 (1) m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.323010:0:261296:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.323012:0:261296:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.323014:0:261296:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323016:0:261296:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323018:0:261296:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1148 -> ffff9c37313e1148 00000080:00200000:0.0:1788572470.323020:0:261296:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xda0:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.323022:0:261296:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.323025:0:261296:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.323027:0:261296:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50000 (ffff9c3718da6cf8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.323030:0:261296:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.323031:0:261296:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50000 (ffff9c3718da6cf8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.323035:0:261296:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.323039:0:261296:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.323041:0:261296:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293059160 : -109714416492456 : ffff9c371f2da058) 00000020:00000001:0.0:1788572470.323043:0:261296:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.323044:0:261296:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cf6b0 (ffff9c3718da6758/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.323047:0:261296:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.323048:0:261296:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.323049:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.323050:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.323052:0:261296:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.323053:0:261296:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.323054:0:261296:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.323055:0:261296:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.323056:0:261296:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.323058:0:261296:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50000 (ffff9c3718da6cf8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.323063:0:261296:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.323064:0:261296:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.323065:0:261296:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cf6b0 (ffff9c3718da6cf8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.323175:0:261296:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb8390. 00000020:00000001:0.0:1788572470.323179:0:261296:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.323181:0:261296:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cf680. 00020000:00000001:0.0:1788572470.323184:0:261296:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.323185:0:261296:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.323186:0:261296:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.323188:0:261296:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323191:0:261296:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.323193:0:261296:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.323195:0:261296:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.323196:0:261296:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.323198:0:261296:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.323200:0:261296:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6750 00020000:00000001:0.0:1788572470.323203:0:261296:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.323205:0:261296:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.323207:0:261296:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.323210:0:261296:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda0:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.323214:0:261296:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.323216:0:261296:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.323218:0:261296:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.323220:0:261296:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.323222:0:261296:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.323223:0:261296:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6cf0 00000001:00000001:0.0:1788572470.323225:0:261296:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323232:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7d08 00000080:00000008:0.0:1788572470.323234:0:261296:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-012:[0x200002b17:0xda0:0x0](ffff9c372f0faa08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.323240:0:261296:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.323242:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7da0 00000080:00002000:0.0:1788572470.323244:0:261296:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-012 (ffff9c370833c0d8, parent ffff9c3719d93af8, inode ffff9c372f0faa08) hashed, 00000080:00000001:0.0:1788572470.323247:0:261296:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.323277:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.323280:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.323282:0:261296:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.323284:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7bc0 00000080:00200000:0.0:1788572470.323286:0:261296:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.323292:0:261296:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572470.323298:0:261296:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.323303:0:261296:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.323306:0:261296:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.323309:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7ae0 00800000:00000002:0.0:1788572470.323311:0:261296:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.323314:0:261296:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.323317:0:261296:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.323319:0:261296:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.323321:0:261296:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.323324:0:261296:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.323325:0:261296:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.323327:0:261296:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.323329:0:261296:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.323332:0:261296:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.323337:0:261296:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.323338:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca79b0 00000002:00010000:0.0:1788572470.323340:0:261296:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.323343:0:261296:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.323345:0:261296:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.323348:0:261296:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.323354:0:261296:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.323357:0:261296:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.323361:0:261296:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.323364:0:261296:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.323368:0:261296:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 02000000:00000001:0.0:1788572470.323377:0:261296:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.323382:0:261296:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.323386:0:261296:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00000002:00000001:0.0:1788572470.323392:0:261296:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.323394:0:261296:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.323396:0:261296:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.323398:0:261296:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.323399:0:261296:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.323401:0:261296:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.323403:0:261296:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.323406:0:261296:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.323409:0:261296:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572470.323415:0:261296:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.323417:0:261296:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.323420:0:261296:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.323423:0:261296:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.323426:0:261296:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.323428:0:261296:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.323429:0:261296:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.323430:0:261296:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.323432:0:261296:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.323434:0:261296:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.323436:0:261296:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.323439:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.323440:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.323442:0:261296:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.323443:0:261296:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.323445:0:261296:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1ca7bf8 00010000:00000001:0.0:1788572470.323447:0:261296:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.323448:0:261296:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.323449:0:261296:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.323450:0:261296:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.323452:0:261296:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.323454:0:261296:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.323456:0:261296:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00010000:00000001:0.0:1788572470.323461:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.323462:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.323463:0:261296:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.323465:0:261296:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.323467:0:261296:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 124 lru_type: 1 00010000:00000001:0.0:1788572470.323473:0:261296:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.323475:0:261296:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.323476:0:261296:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.323479:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.323479:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.323480:0:261296:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.323483:0:261296:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.323484:0:261296:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.323486:0:261296:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.323489:0:261296:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.323490:0:261296:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.323491:0:261296:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.323492:0:261296:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.323494:0:261296:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.323496:0:261296:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.323498:0:261296:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.323502:0:261296:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.323506:0:261296:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.323508:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7b88 00000080:00002000:0.0:1788572470.323509:0:261296:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 13 00000080:00000001:0.0:1788572470.323512:0:261296:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323514:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7bd0 00000080:00200000:0.0:1788572470.323515:0:261296:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.323520:0:261296:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.323523:0:261296:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.323525:0:261296:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.323526:0:261296:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.323528:0:261296:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.323530:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7be8 00000080:00002000:0.0:1788572470.323531:0:261296:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-012(ffff9c370833c0d8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.323533:0:261296:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.323534:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7c30 00000080:00200000:0.0:1788572470.323536:0:261296:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-012, flags=9 00000080:00000001:0.0:1788572470.323540:0:261296:0:(statahead.c:1997:revalidate_statahead_dentry()) Process entered 00000080:00000020:0.0:1788572470.323543:0:261296:0:(statahead.c:273:sa_get()) lustre: wrong pid=261292:261296 for entry f434.sanity-012 00000080:00000001:0.0:1788572470.323546:0:261296:0:(statahead.c:2046:revalidate_statahead_dentry()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00000080:00400000:0.0:1788572470.323549:0:261296:0:(statahead.c:2622:ll_revalidate_statahead()) revalidate statahead f434.sanity-012: rc = -22. 00000080:00000001:0.0:1788572470.323572:0:261296:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.323575:0:261296:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.323577:0:261296:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.323579:0:261296:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ca7e40 00000080:00002000:0.0:1788572470.323580:0:261296:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-012 (ffff9c370833c0d8, parent ffff9c3719d93af8, inode ffff9c372f0faa08) hashed, 00000080:00000001:0.0:1788572470.323583:0:261296:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.343325:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.343335:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572470.343343:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.343352:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.343359:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1b28 00000400:00000010:0.0:1788572470.343363:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1b28. 00000100:00000001:0.0:1788572470.343368:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.343376:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809933568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.343390:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.343394:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71f80 x1875440809933568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.343401:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.343403:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.348021:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1788572470.354004:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.354009:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.354011:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.354019:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.354029:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.354031:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.354035:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.354039:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.354041:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.354045:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809933696:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.354049:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.354054:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.354056:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.354059:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.354065:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b7e00. 02000000:00000001:0.0:1788572470.354523:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.354532:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.354537:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f20. 00000400:00000010:0.0:1788572470.354546:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572470.354554:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809933696, portal 18 00000100:00000001:0.0:1788572470.354556:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.354558:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788572470.354567:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.354578:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.354581:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.354583:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572470.354586:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809933696, offset 0 00000400:00000200:0.0:1788572470.354592:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.354601:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.354608:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.354614:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.354619:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.354622:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.354625:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.354628:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539674:539674:256:4294967295] 192.168.202.148@tcp LPNI seq info [539639:539639:8:4294967295] 00000400:00000200:0.0:1788572470.354633:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.355421:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.355428:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000200:0.0:1788572470.356153:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.356157:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572470.356162:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.356168:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.356173:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1908 00000400:00000010:0.0:1788572470.356175:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1908. 00000100:00000001:0.0:1788572470.356179:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.356184:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809933824/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.356193:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.356197:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809933824/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.356203:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.356205:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.359043:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.359065:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.359081:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.359084:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.359095:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.359105:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.359116:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.359122:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.359127:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.359131:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41745fc00 00000400:00000200:0.0:1788572470.359138:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.359144:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529999 [1] + 224 00000800:00000001:0.0:1788572470.359151:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.359159:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.359162:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.359168:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.359175:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.359185:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809933824/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.359198:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809933824/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.359220:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.359232:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.359236:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1788572470.363702:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.363717:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.363723:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.363727:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.363737:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.363743:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15010 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.363749:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.363752:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609383 00000100:00000001:0.0:1788572470.363756:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609544 00000100:00000001:0.0:1788572470.363758:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609383 00000100:00000001:0.0:1788572470.363761:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609544 00000100:00000001:0.0:1788572470.363763:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.363767:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.363772:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.363775:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.363784:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.363786:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572470.363789:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.363790:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.363792:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.363795:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.365307:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.365315:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572470.365561:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.365570:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.365577:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31bb0 00000400:00000010:0.0:1788572470.365580:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31bb0. 00000100:00000001:0.0:1788572470.365586:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.365594:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.365606:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.365610:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.365618:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.365619:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.386037:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.386052:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.386056:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.386058:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.386078:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.386087:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.386096:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.386102:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.386107:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.386112:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41745fb80 00000400:00000200:0.0:1788572470.386117:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.386123:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529989 [1] + 224 00000800:00000001:0.0:1788572470.386131:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.386138:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.386141:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.386145:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.386152:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.386160:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.386172:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c5c00 x1875440809933696/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.386190:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.386198:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.386200:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.394743:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.394748:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572470.394752:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.394757:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.394762:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1660 00000400:00000010:0.0:1788572470.394765:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1660. 00000100:00000001:0.0:1788572470.394769:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.394772:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809934080/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.394781:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.394783:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809934080/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.394789:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.394791:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.399940:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.399955:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.399959:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.399961:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.399970:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.399977:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.399985:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.399990:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.399995:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.399999:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41745fd00 00000400:00000200:0.0:1788572470.400005:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.400011:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x5299a9 [1] + 224 00000800:00000001:0.0:1788572470.400018:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.400028:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.400032:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.400037:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.400044:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.400053:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809934080/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.400075:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809934080/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.400096:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.400104:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.400107:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.415595:0:261302:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.415602:0:261302:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.415605:0:261302:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.415608:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fb20 00000080:00200000:0.0:1788572470.415613:0:261302:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.415621:0:261302:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.415628:0:261302:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.415632:0:261302:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.415666:0:261302:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.415669:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fa40 00800000:00000002:0.0:1788572470.415673:0:261302:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.415677:0:261302:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.415679:0:261302:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.415682:0:261302:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.415684:0:261302:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.415686:0:261302:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.415689:0:261302:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.415691:0:261302:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.415693:0:261302:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.415697:0:261302:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.415702:0:261302:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.415704:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f910 00000002:00010000:0.0:1788572470.415706:0:261302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.415710:0:261302:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.415712:0:261302:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.415715:0:261302:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.415722:0:261302:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.415725:0:261302:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.415730:0:261302:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.415733:0:261302:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.415738:0:261302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 128 lru_type: 1 02000000:00000001:0.0:1788572470.415750:0:261302:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.415754:0:261302:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.415758:0:261302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 128 lru_type: 1 00000002:00000001:0.0:1788572470.415766:0:261302:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.415768:0:261302:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.415770:0:261302:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.415772:0:261302:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.415774:0:261302:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.415776:0:261302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.415779:0:261302:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.415782:0:261302:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.415785:0:261302:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572470.415793:0:261302:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.415794:0:261302:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.415797:0:261302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.415801:0:261302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.415804:0:261302:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.415807:0:261302:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.418915:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.418925:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.418930:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.418934:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.418938:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.418943:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.418945:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.418947:0:261302:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.418950:0:261302:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.418954:0:261302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d1fb58 00010000:00000001:0.0:1788572470.418956:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.418958:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.418959:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.418961:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.418964:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.418966:0:261302:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.418971:0:261302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 128 lru_type: 1 00010000:00000001:0.0:1788572470.418980:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.418981:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.418983:0:261302:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.418986:0:261302:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.418989:0:261302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 129 lru_type: 1 00010000:00000001:0.0:1788572470.419002:0:261302:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.419006:0:261302:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.419008:0:261302:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.419011:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.419012:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.419014:0:261302:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419017:0:261302:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.419019:0:261302:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419021:0:261302:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419024:0:261302:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419026:0:261302:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.419027:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.419028:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.419031:0:261302:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.419033:0:261302:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.419037:0:261302:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.419043:0:261302:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.419047:0:261302:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.419050:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fae8 00000080:00002000:0.0:1788572470.419052:0:261302:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 15 00000080:00000001:0.0:1788572470.419055:0:261302:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.419057:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fb30 00000080:00200000:0.0:1788572470.419058:0:261302:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.419064:0:261302:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.419068:0:261302:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.419070:0:261302:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.419073:0:261302:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.419081:0:261302:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fde0. 00000020:00000001:0.0:1788572470.419084:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fae8 00000080:00200000:0.0:1788572470.419087:0:261302:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-015, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.419090:0:261302:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.419092:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f998 00000080:00200000:0.0:1788572470.419094:0:261302:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-015, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000001:0.0:1788572470.419099:0:261302:0:(statahead.c:1997:revalidate_statahead_dentry()) Process entered 00000080:00000020:0.0:1788572470.419104:0:261302:0:(statahead.c:273:sa_get()) lustre: wrong pid=261292:261302 for entry f434.sanity-015 00000080:00000001:0.0:1788572470.419107:0:261302:0:(statahead.c:2046:revalidate_statahead_dentry()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00000080:00400000:0.0:1788572470.419110:0:261302:0:(statahead.c:2622:ll_revalidate_statahead()) revalidate statahead f434.sanity-015: rc = -22. 00000080:00000010:0.0:1788572470.419118:0:261302:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.419122:0:261302:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.419126:0:261302:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.419129:0:261302:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.419132:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f8b8 00800000:00000002:0.0:1788572470.419134:0:261302:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-015' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.419137:0:261302:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.419141:0:261302:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.419144:0:261302:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.419147:0:261302:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.419149:0:261302:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.419150:0:261302:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.419153:0:261302:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.419155:0:261302:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.419159:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f850 00800000:00000002:0.0:1788572470.419161:0:261302:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-015' -> mds #0 00000002:00000001:0.0:1788572470.419166:0:261302:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.419168:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f788 00000002:00010000:0.0:1788572470.419170:0:261302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-015,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.419174:0:261302:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.419176:0:261302:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.419181:0:261302:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572470.419184:0:261302:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=44 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.419190:0:261302:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.419193:0:261302:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.419196:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.419197:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.419199:0:261302:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.419201:0:261302:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.419204:0:261302:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.419208:0:261302:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.419214:0:261302:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572470.419223:0:261302:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419228:0:261302:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.419230:0:261302:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419232:0:261302:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.419233:0:261302:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.419235:0:261302:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.419243:0:261302:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00010000:00000001:0.0:1788572470.419246:0:261302:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.419249:0:261302:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.419255:0:261302:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.419258:0:261302:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.419261:0:261302:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.419264:0:261302:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6ba00. 00000020:00000001:0.0:1788572470.419268:0:261302:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.419271:0:261302:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6ba00 with handle 0x14415860931683b9 to hash 00000020:00000001:0.0:1788572470.419273:0:261302:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.419274:0:261302:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.419278:0:261302:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.419282:0:261302:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.419284:0:261302:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.419288:0:261302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.419297:0:261302:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.419307:0:261302:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261302 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.419312:0:261302:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.419314:0:261302:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.419320:0:261302:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.419322:0:261302:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572470.419325:0:261302:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.419326:0:261302:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788572470.419330:0:261302:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.419334:0:261302:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.419336:0:261302:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.419338:0:261302:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.419348:0:261302:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.419357:0:261302:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.419359:0:261302:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.419363:0:261302:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.419366:0:261302:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.419368:0:261302:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.419372:0:261302:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261302:1875440809934336:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.419376:0:261302:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.419380:0:261302:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.419383:0:261302:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.419385:0:261302:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.419391:0:261302:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ed000. 02000000:00000001:0.0:1788572470.419392:0:261302:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.419401:0:261302:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.419405:0:261302:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89f8. 00000400:00000010:0.0:1788572470.419412:0:261302:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572470.419420:0:261302:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809934336, portal 10 00000100:00000001:0.0:1788572470.419422:0:261302:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.419423:0:261302:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572470.419429:0:261302:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.419436:0:261302:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.419439:0:261302:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.419441:0:261302:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572470.419443:0:261302:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809934336, offset 0 00000400:00000200:0.0:1788572470.419450:0:261302:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.419459:0:261302:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.419467:0:261302:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.419473:0:261302:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.419476:0:261302:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.419480:0:261302:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.419483:0:261302:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.419487:0:261302:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539679:539679:256:4294967295] 192.168.202.148@tcp LPNI seq info [539644:539644:8:4294967295] 00000400:00000200:0.0:1788572470.419491:0:261302:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.419501:0:261302:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.419506:0:261302:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.419512:0:261302:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.419518:0:261302:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.419526:0:261302:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.419530:0:261302:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.419546:0:261302:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.419550:0:261302:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15018 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.419554:0:261302:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.419557:0:261302:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609440 00000100:00000001:0.0:1788572470.419585:0:261302:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609573 00000100:00000001:0.0:1788572470.419589:0:261302:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609440 00000100:00000001:0.0:1788572470.419591:0:261302:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609573 00000100:00000001:0.0:1788572470.419593:0:261302:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.419595:0:261302:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.419597:0:261302:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.419599:0:261302:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.419602:0:261302:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.419605:0:261302:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.419608:0:261302:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.430352:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.430363:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572470.430372:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.430381:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.430390:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788572470.430393:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788572470.430399:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.430406:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.430419:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.430423:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.430429:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.430431:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.433185:0:261302:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.433193:0:261302:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.433196:0:261302:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.433199:0:261302:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89f8. 00000400:00000200:0.0:1788572470.433204:0:261302:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31bb0 00000400:00000010:0.0:1788572470.433206:0:261302:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31bb0. 00000100:00000001:0.0:1788572470.433209:0:261302:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.433214:0:261302:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.433224:0:261302:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.433230:0:261302:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.433231:0:261302:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.433234:0:261302:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.433237:0:261302:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.433239:0:261302:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.433242:0:261302:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.433245:0:261302:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.433254:0:261302:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=34 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572470.433259:0:261302:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.433262:0:261302:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.433263:0:261302:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.433266:0:261302:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.433270:0:261302:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.433274:0:261302:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.433276:0:261302:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.433278:0:261302:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.433281:0:261302:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.433286:0:261302:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.433292:0:261302:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.433295:0:261302:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.433296:0:261302:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.433300:0:261302:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.433307:0:261302:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261302:1875440809934336:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.433312:0:261302:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.433315:0:261302:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.433317:0:261302:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572470.433318:0:261302:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.433322:0:261302:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.433329:0:261302:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572470.433332:0:261302:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.433336:0:261302:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572470.433338:0:261302:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.433341:0:261302:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.433345:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.433348:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.433351:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=4 00000020:00000001:0.0:1788572470.433354:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433357:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000040:0.0:1788572470.433363:0:261302:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6ba00, remote cookie: 0xd9894672d99a2076, flags: 0x1001 00010000:00010000:0.0:1788572470.433368:0:261302:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.433378:0:261302:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xda3:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572470.433381:0:261302:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572470.433388:0:261302:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25500 count: 3 00010000:00000001:0.0:1788572470.433390:0:261302:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.433393:0:261302:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.433397:0:261302:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.433405:0:261302:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433412:0:261302:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.433416:0:261302:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572470.433418:0:261302:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572470.433420:0:261302:0:(ldlm_lock.c:1024:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572470.433421:0:261302:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572470.433424:0:261302:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xda3:0x0].0x0 (ffff9c3730c25500) refcount = 3 00010000:00000040:0.0:1788572470.433427:0:261302:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.433429:0:261302:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320efe00/0x144158609316832d lrc: 1/0,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ea1 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000040:0.0:1788572470.433436:0:261302:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731f15400/0x1441586093167d99 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xda3:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a0b6f expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572470.433443:0:261302:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433449:0:261302:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572470.433452:0:261302:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.433454:0:261302:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.433457:0:261302:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.433459:0:261302:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.433462:0:261302:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433467:0:261302:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.433469:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433470:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.433471:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433472:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.433474:0:261302:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.433478:0:261302:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572470.433480:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.433481:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.433483:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=3 00000020:00000001:0.0:1788572470.433485:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433487:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433490:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.433491:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.433492:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=4 00000020:00000001:0.0:1788572470.433494:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433496:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433498:0:261302:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.433501:0:261302:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.433504:0:261302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433510:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433511:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.433512:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.433514:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.433515:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=5 00000020:00000001:0.0:1788572470.433517:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433519:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433521:0:261302:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.433524:0:261302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433530:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433531:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.433534:0:261302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.433540:0:261302:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.433542:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433543:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.433544:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.433545:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572470.433969:0:261302:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572470.433983:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.433984:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.433986:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=3 00000020:00000001:0.0:1788572470.433988:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.433990:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00000002:00010000:0.0:1788572470.433994:0:261302:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572470.434007:0:261302:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091cc0. 00010000:00000001:0.0:1788572470.434009:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434011:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.434012:0:261302:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.434014:0:261302:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.434017:0:261302:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572470.434019:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434020:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434022:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=3 00000020:00000001:0.0:1788572470.434023:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.434025:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00000002:00010000:0.0:1788572470.434028:0:261302:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.434034:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434036:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.434037:0:261302:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572470.434038:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434039:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434041:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=3 00000020:00000001:0.0:1788572470.434042:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.434045:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000040:0.0:1788572470.434049:0:261302:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25500 count: 4 00010000:00000001:0.0:1788572470.434051:0:261302:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.434055:0:261302:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.434058:0:261302:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.434061:0:261302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320efe00/0x144158609316832d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ea1 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 02000000:00000001:0.0:1788572470.434143:0:261302:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.434149:0:261302:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.434152:0:261302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320efe00/0x144158609316832d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ea1 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.434158:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434159:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.434161:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434162:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434164:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=3 00000020:00000001:0.0:1788572470.434165:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.434168:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572470.434169:0:261302:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:0.0:1788572470.434172:0:261302:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.434177:0:261302:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.434179:0:261302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.434183:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434184:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.434186:0:261302:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ba00/0x14415860931683b9 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a2076 expref: -99 pid: 261302 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.434191:0:261302:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.434193:0:261302:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434196:0:261302:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:0.0:1788572470.434200:0:261302:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff9c371eeb5300. 00010000:00000001:0.0:1788572470.434203:0:261302:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:0.0:1788572470.434222:0:261302:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:0.0:1788572470.434225:0:261302:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434227:0:261302:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434228:0:261302:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.434230:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434231:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.434232:0:261302:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572470.434236:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1f710 00000002:00002000:0.0:1788572470.434238:0:261302:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-015 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572470.434241:0:261302:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.434246:0:261302:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.434248:0:261302:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572470.434252:0:261302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572470.434255:0:261302:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.434259:0:261302:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.434262:0:261302:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572470.434264:0:261302:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1d1fb20 it_disposition b 00000080:00000001:0.0:1788572470.434267:0:261302:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572470.434273:0:261302:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572470.434276:0:261302:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572470.434279:0:261302:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329059 : 144115373145329059 : 200002b17000da3) 00000080:00000001:0.0:1788572470.434282:0:261302:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572470.434289:0:261302:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.434292:0:261302:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943224 : -109714522608392 : ffff9c3718da6cf8) 00000020:00001000:0.0:1788572470.434295:0:261302:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6cf0 00000020:00000001:0.0:1788572470.434297:0:261302:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.434301:0:261302:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.434303:0:261302:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.434309:0:261302:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.434314:0:261302:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.434318:0:261302:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572470.434320:0:261302:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572470.434323:0:261302:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xda3:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.434328:0:261302:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.434332:0:261302:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20976 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.434336:0:261302:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711aced90) owner ffff9c372eb94f80 00020000:00000001:0.0:1788572470.434339:0:261302:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.434341:0:261302:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711aced90) owner ffff9c372eb94f80 00020000:00000010:0.0:1788572470.434345:0:261302:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.434349:0:261302:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.434352:0:261302:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572470.434355:0:261302:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xda3:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.434357:0:261302:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.434358:0:261302:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572470.434361:0:261302:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6cf0 00000080:00000001:0.0:1788572470.434365:0:261302:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329059 : 144115373145329059 : 200002b17000da3) 00000080:00000001:0.0:1788572470.434367:0:261302:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572470.434372:0:261302:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xda3:0x0](ffff9c372f0fba88): rc = 0 00000080:00000001:0.0:1788572470.434376:0:261302:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359559535240 : -109714150016376 : ffff9c372f0fba88) 00010000:00000001:0.0:1788572470.434378:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434379:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434381:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320efe00 refcount=3 00000020:00000001:0.0:1788572470.434383:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00010000:00000001:0.0:1788572470.434386:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00000080:00000001:0.0:1788572470.434389:0:261302:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572470.434391:0:261302:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.434392:0:261302:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943224 : -109714522608392 : ffff9c3718da6cf8) 00000020:00001000:0.0:1788572470.434394:0:261302:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6cf0 00000020:00000001:0.0:1788572470.434395:0:261302:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.434396:0:261302:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.434398:0:261302:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.434400:0:261302:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.434403:0:261302:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.434405:0:261302:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572470.434407:0:261302:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572470.434409:0:261302:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xda3:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.434412:0:261302:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.434415:0:261302:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20976 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.434417:0:261302:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711aced90) owner ffff9c372eb94f80 00020000:00000001:0.0:1788572470.434419:0:261302:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.434421:0:261302:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711aced90) owner ffff9c372eb94f80 00020000:00000010:0.0:1788572470.434423:0:261302:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.434426:0:261302:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.434427:0:261302:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572470.434428:0:261302:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xda3:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.434430:0:261302:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.434431:0:261302:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.434434:0:261302:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572470.434437:0:261302:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711aced90) owner 0000000000000000/ffff9c372eb94f80 00020000:00000002:0.0:1788572470.434440:0:261302:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3711bdd0c0 addref 2/0 by ffff9c372eb94f80. 00020000:00000002:0.0:1788572470.434442:0:261302:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711aced90) owner 0000000000000000/ffff9c372eb94f80 00020000:00000001:0.0:1788572470.434445:0:261302:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572470.434447:0:261302:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572470.434448:0:261302:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572470.434449:0:261302:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572470.434451:0:261302:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572470.434453:0:261302:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xda3:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572470.434455:0:261302:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6cf0 00000080:00000001:0.0:1788572470.434458:0:261302:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434459:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434460:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.434462:0:261302:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572470.434464:0:261302:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572470.434466:0:261302:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.434468:0:261302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xda3:0x0](ffff9c372f0fba88) for lock 0x144158609316832d 00800000:00000001:0.0:1788572470.434471:0:261302:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.434473:0:261302:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.434474:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434475:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434476:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320efe00 refcount=3 00000020:00000001:0.0:1788572470.434478:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00010000:00000001:0.0:1788572470.434480:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00010000:00000001:0.0:1788572470.434482:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434483:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.434484:0:261302:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.434486:0:261302:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572470.434492:0:261302:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c370833c870 inode ffff9c372f0fba88 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572470.434496:0:261302:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572470.434499:0:261302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xda3:0x0](ffff9c372f0fba88) 00000080:00010000:0.0:1788572470.434503:0:261302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316832d from it ffffb95bc1d1fb20 00010000:00000001:0.0:1788572470.434505:0:261302:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.434506:0:261302:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.434507:0:261302:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320efe00 refcount=3 00000020:00000001:0.0:1788572470.434509:0:261302:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00010000:00000001:0.0:1788572470.434510:0:261302:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609818624 : -109714099732992 : ffff9c37320efe00) 00010000:00000001:0.0:1788572470.434512:0:261302:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.434514:0:261302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320efe00/0x144158609316832d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1ea1 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.434520:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434521:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.434523:0:261302:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.434525:0:261302:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.434527:0:261302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320efe00/0x144158609316832d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ea1 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00000001:0.0:1788572470.434534:0:261302:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.434537:0:261302:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.434540:0:261302:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.434544:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434545:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.434547:0:261302:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434549:0:261302:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.434551:0:261302:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434553:0:261302:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434556:0:261302:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.434558:0:261302:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.434559:0:261302:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.434560:0:261302:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.434562:0:261302:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572470.434565:0:261302:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000100:00000001:0.0:1788572470.434573:0:261302:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.434581:0:261302:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440809934336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.434589:0:261302:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.434591:0:261302:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.434594:0:261302:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ed000. 02000000:00000001:0.0:1788572470.434597:0:261302:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.434600:0:261302:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.434602:0:261302:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.434605:0:261302:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.434607:0:261302:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20f400. 02000000:00000001:0.0:1788572470.434611:0:261302:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.434613:0:261302:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.434614:0:261302:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788572470.434617:0:261302:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.434618:0:261302:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.434621:0:261302:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572470.434622:0:261302:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1d1fb20 released 00000080:00000001:0.0:1788572470.434623:0:261302:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572470.434975:0:261302:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d1fd08 00000080:00200008:0.0:1788572470.434979:0:261302:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-015:[0x200002b17:0xda3:0x0](ffff9c372f0fba88), request_mask 2047, flags 0, foreign 0 00000800:00000200:0.0:1788572470.437307:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.437312:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572470.437318:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.437324:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.437329:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1c38 00000400:00000010:0.0:1788572470.437332:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1c38. 00000100:00000001:0.0:1788572470.437336:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.437341:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69c00 x1875440809934464/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.437350:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.437353:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440809934464/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.437359:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.437360:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572470.439742:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.439754:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572470.439766:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.439775:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.439782:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572470.439785:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572470.439791:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.439799:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.440493:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.440503:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.440512:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.440514:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.443681:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.443686:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.443693:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.443695:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.443698:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2e18. 00000400:00000200:0.0:1788572470.443703:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572470.443706:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572470.443709:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.443713:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.443723:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.443730:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.443732:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.443734:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.443738:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.443740:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.443743:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.443746:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.443756:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=34 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.443761:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=7 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572470.443764:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.443766:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.443769:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.443773:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.443776:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.443778:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572470.443780:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.443783:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.443789:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.443801:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.443804:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.443806:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572470.443818:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572470.443821:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.443823:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.443826:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=4 00000020:00000001:0.0:1788572470.443829:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443832:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443834:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.443835:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.443836:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=5 00000020:00000001:0.0:1788572470.443838:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443839:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443843:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.443847:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.443851:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.443862:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.443863:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.443865:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.443867:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.443868:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.443869:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=6 00000020:00000001:0.0:1788572470.443871:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443890:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000040:0.0:1788572470.443897:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204ae00, remote cookie: 0xd9894672d99a207d, flags: 0x1001 00010000:00010000:0.0:1788572470.443902:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.443914:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572470.443917:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.443919:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.443922:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572470.443926:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.443934:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51f0:0x0].0x0 (ffff9c37039e1700) refcount = 1 00010000:00000040:0.0:1788572470.443936:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.443939:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.443947:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.443949:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.443952:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.443954:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.443958:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.443964:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.443965:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.443967:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.443968:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.443969:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.443971:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572470.443974:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572470.443977:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.443978:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.443979:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=5 00000020:00000001:0.0:1788572470.443981:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.443983:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00000008:00000001:0.0:1788572470.443988:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572470.443992:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9dc0 -> ffff9c371ee2e788 00000008:00010000:0.0:1788572470.443998:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.444006:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9dc0 -> ffff9c371ee2e788 00000020:00000001:0.0:1788572470.444008:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572470.444011:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572470.444014:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572470.444017:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.444019:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.444020:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9dc0 -> ffff9c371ee2e788 00000008:00000001:0.0:1788572470.444022:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572470.444025:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.444027:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572470.444041:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572470.444043:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444045:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.444047:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.444048:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=6 00000020:00000001:0.0:1788572470.444050:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.444053:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.444055:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.444058:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.444064:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444066:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.444069:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.444087:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.444089:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444091:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.444092:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444094:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.444095:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.444096:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204ae00 refcount=5 00000020:00000001:0.0:1788572470.444098:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.444100:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609142784 : -109714100408832 : ffff9c373204ae00) 00010000:00000001:0.0:1788572470.444102:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.444104:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.444111:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444112:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.444113:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.444117:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.444119:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000004a4993f9/0x14415860931683c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a207d expref: -99 pid: 261302 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572470.444128:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.444131:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.444133:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.444137:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444138:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.444140:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444143:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.444146:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444149:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444151:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.444153:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.444154:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444155:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.444156:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.444157:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.444159:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.444165:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.444176:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809934592:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.444182:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.444185:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.444189:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440809934592/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.444197:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.444200:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.444205:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe97800. 02000000:00000001:0.0:1788572470.444211:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.444214:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.444216:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572470.444218:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.444221:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711214600. 02000000:00000001:0.0:1788572470.444225:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.444227:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.444229:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788572470.444232:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.444234:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.444236:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.444244:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.444246:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.444249:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.444251:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.470205:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.470215:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572470.470225:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.470233:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.470240:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3880 00000400:00000010:0.0:1788572470.470243:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3880. 00000100:00000001:0.0:1788572470.470249:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.470257:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6680 x1875440809934720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.470271:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.470275:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6680 x1875440809934720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.470282:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.470284:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.471971:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.471986:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.471991:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.471993:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.472000:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.472008:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.472017:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.472021:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.472026:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.472030:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b41745ff80 00000400:00000200:0.0:1788572470.472035:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.472041:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x5299e1 [1] + 224 00000800:00000001:0.0:1788572470.472048:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.472054:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.472058:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.472062:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.472077:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.472085:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6680 x1875440809934720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.472095:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6680 x1875440809934720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.472107:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.472114:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.472117:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.475332:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.475336:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5a00. 00000400:00000200:0.0:1788572470.475342:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.475347:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.475354:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c16e8 00000400:00000010:0.0:1788572470.475356:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c16e8. 00000100:00000001:0.0:1788572470.475361:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.475365:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809934848/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.475376:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.475379:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809934848/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.475386:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.475388:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572470.475455:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.475458:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572470.475461:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.475464:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.475467:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1f68 00000400:00000010:0.0:1788572470.475469:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1f68. 00000100:00000001:0.0:1788572470.475472:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.475475:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b100 x1875440809934976/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.475482:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.475485:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440809934976/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.475490:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.475492:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.478034:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.478051:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.478056:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.478058:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.478067:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.478075:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.478083:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.478088:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.478093:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.478098:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460000 00000400:00000200:0.0:1788572470.478103:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.478108:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x5299f1 [1] + 224 00000800:00000001:0.0:1788572470.478115:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.478122:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.478125:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.478130:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.478136:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.478145:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809934848/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.478154:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809934848/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.478174:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.478182:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.478185:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.498971:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.498979:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.498982:0:261306:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.498986:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03b20 00000080:00200000:0.0:1788572470.498991:0:261306:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.498999:0:261306:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.499007:0:261306:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.499012:0:261306:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.499017:0:261306:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.499020:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03a40 00800000:00000002:0.0:1788572470.499022:0:261306:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.499027:0:261306:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.499030:0:261306:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.499034:0:261306:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.499036:0:261306:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.499039:0:261306:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.499042:0:261306:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.499044:0:261306:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.499047:0:261306:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.499051:0:261306:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.499056:0:261306:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.499058:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03910 00000002:00010000:0.0:1788572470.499060:0:261306:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.499064:0:261306:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.499066:0:261306:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.499070:0:261306:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.499078:0:261306:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.499082:0:261306:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.499088:0:261306:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.499093:0:261306:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.499097:0:261306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 132 lru_type: 1 02000000:00000001:0.0:1788572470.499108:0:261306:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.499113:0:261306:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.499116:0:261306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 132 lru_type: 1 00000002:00000001:0.0:1788572470.499123:0:261306:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.499125:0:261306:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.499127:0:261306:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.499129:0:261306:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.499131:0:261306:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.499134:0:261306:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.499136:0:261306:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.499139:0:261306:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.499142:0:261306:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572470.499148:0:261306:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.499150:0:261306:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.499153:0:261306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.499156:0:261306:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.499159:0:261306:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.499162:0:261306:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.499164:0:261306:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.499166:0:261306:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.499169:0:261306:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.499171:0:261306:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.499174:0:261306:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.499177:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.499178:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.499180:0:261306:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.499181:0:261306:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.499184:0:261306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a03b58 00010000:00000001:0.0:1788572470.499186:0:261306:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.499187:0:261306:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.499188:0:261306:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.499190:0:261306:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.499192:0:261306:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.499194:0:261306:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.499197:0:261306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 132 lru_type: 1 00010000:00000001:0.0:1788572470.499204:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.499205:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.499207:0:261306:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.499209:0:261306:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.499212:0:261306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 133 lru_type: 1 00010000:00000001:0.0:1788572470.499219:0:261306:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.499222:0:261306:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.499225:0:261306:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.499231:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.499232:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.499234:0:261306:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499237:0:261306:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.499239:0:261306:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499242:0:261306:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499245:0:261306:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499247:0:261306:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.499248:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.499249:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.499252:0:261306:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.499254:0:261306:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.499256:0:261306:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.499261:0:261306:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.499265:0:261306:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.499268:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03ae8 00000080:00002000:0.0:1788572470.499269:0:261306:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 16 00000080:00000001:0.0:1788572470.499272:0:261306:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.499275:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03b30 00000080:00200000:0.0:1788572470.499276:0:261306:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.499281:0:261306:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.499285:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.499286:0:261306:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.499289:0:261306:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.499297:0:261306:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f800. 00000020:00000001:0.0:1788572470.499301:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03ae8 00000080:00200000:0.0:1788572470.499303:0:261306:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-017, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.499306:0:261306:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.499308:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03998 00000080:00200000:0.0:1788572470.499310:0:261306:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-017, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.499317:0:261306:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.499320:0:261306:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.499324:0:261306:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.499326:0:261306:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.499327:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a038b8 00800000:00000002:0.0:1788572470.499329:0:261306:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-017' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.499332:0:261306:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.499335:0:261306:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.499336:0:261306:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.499337:0:261306:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.499339:0:261306:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.499341:0:261306:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.499343:0:261306:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.499344:0:261306:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.499347:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03850 00800000:00000002:0.0:1788572470.499348:0:261306:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-017' -> mds #0 00000002:00000001:0.0:1788572470.499351:0:261306:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.499352:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03788 00000002:00010000:0.0:1788572470.499353:0:261306:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-017,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.499356:0:261306:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.499359:0:261306:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.499363:0:261306:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572470.499367:0:261306:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.499373:0:261306:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.499377:0:261306:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.499380:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.499382:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.499383:0:261306:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.499386:0:261306:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.499387:0:261306:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.499391:0:261306:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.499397:0:261306:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572470.499404:0:261306:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499410:0:261306:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.499412:0:261306:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499414:0:261306:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.499415:0:261306:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.499417:0:261306:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.499423:0:261306:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00010000:00000001:0.0:1788572470.499426:0:261306:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.499429:0:261306:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.499432:0:261306:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.499434:0:261306:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.499437:0:261306:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.499440:0:261306:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6ac00. 00000020:00000001:0.0:1788572470.499444:0:261306:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.499447:0:261306:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6ac00 with handle 0x14415860931683d5 to hash 00000020:00000001:0.0:1788572470.499448:0:261306:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.499450:0:261306:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572470.499454:0:261306:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572470.499456:0:261306:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.499458:0:261306:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.499461:0:261306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261306 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.499469:0:261306:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261306 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.499479:0:261306:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261306 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.499485:0:261306:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.499487:0:261306:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.499493:0:261306:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572470.499495:0:261306:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572470.499498:0:261306:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.499499:0:261306:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788572470.499503:0:261306:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.499507:0:261306:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.499509:0:261306:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.499510:0:261306:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.499520:0:261306:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809935104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.499528:0:261306:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.499530:0:261306:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.499533:0:261306:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.499536:0:261306:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.499538:0:261306:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.499542:0:261306:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261306:1875440809935104:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.499548:0:261306:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.499553:0:261306:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.499555:0:261306:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.499558:0:261306:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.499562:0:261306:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ed000. 02000000:00000001:0.0:1788572470.499564:0:261306:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.499572:0:261306:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.499575:0:261306:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89f8. 00000400:00000010:0.0:1788572470.499582:0:261306:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572470.499590:0:261306:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809935104, portal 10 00000100:00000001:0.0:1788572470.499592:0:261306:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.499593:0:261306:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572470.499598:0:261306:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4700 x1875440809935104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.499606:0:261306:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.499608:0:261306:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.499611:0:261306:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572470.499614:0:261306:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809935104, offset 0 00000400:00000200:0.0:1788572470.499620:0:261306:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.499629:0:261306:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.499679:0:261306:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.499684:0:261306:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.499688:0:261306:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.499692:0:261306:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.499694:0:261306:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.499698:0:261306:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539685:539685:256:4294967295] 192.168.202.148@tcp LPNI seq info [539650:539650:8:4294967295] 00000400:00000200:0.0:1788572470.499703:0:261306:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.499713:0:261306:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.499719:0:261306:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.499726:0:261306:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572470.499733:0:261306:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.499742:0:261306:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.499747:0:261306:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.499766:0:261306:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.499770:0:261306:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15030 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.499774:0:261306:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.499776:0:261306:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609522 00000100:00000001:0.0:1788572470.499779:0:261306:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609650 00000100:00000001:0.0:1788572470.499781:0:261306:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609522 00000100:00000001:0.0:1788572470.499784:0:261306:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609650 00000100:00000001:0.0:1788572470.499786:0:261306:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.499788:0:261306:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.499791:0:261306:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.499793:0:261306:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.499795:0:261306:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.499798:0:261306:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.499800:0:261306:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.508353:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.508370:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.508374:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.508376:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.508384:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.508392:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.508403:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.508408:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.508414:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.508418:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417460100 00000400:00000200:0.0:1788572470.508423:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.508430:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529a01 [1] + 224 00000800:00000001:0.0:1788572470.508438:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.508444:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.508447:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.508452:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.508462:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.508473:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4700 x1875440809935104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.508485:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4700 x1875440809935104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.508507:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.508517:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.508521:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.510316:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.510320:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510322:0:237083:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.510325:0:237083:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.510328:0:237083:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.510333:0:237083:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.510342:0:237083:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372eb6ac00 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.510346:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.510350:0:237083:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.510357:0:237083:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.510359:0:237083:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.510363:0:237083:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.510369:0:237083:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.510372:0:237083:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.510373:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.510375:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.510377:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572470.510379:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572470.510382:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572470.510385:0:237083:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.510388:0:237083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.510394:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.510398:0:237083:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.510400:0:237083:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.510402:0:237083:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xda5:0x0] mode (null) 00000002:00000001:0.0:1788572470.510404:0:237083:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.510406:0:237083:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.510410:0:237083:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514900 count: 4 00010000:00000001:0.0:1788572470.510412:0:237083:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.510415:0:237083:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.510418:0:237083:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.510420:0:237083:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.510423:0:237083:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510426:0:237083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37320ef600/0x144158609316833b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1ed9 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00000001:0.0:1788572470.510432:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.510433:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.510434:0:237083:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.510436:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.510437:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.510438:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37320ef600 refcount=2 00000020:00000001:0.0:1788572470.510440:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609816576 : -109714099735040 : ffff9c37320ef600) 00010000:00000001:0.0:1788572470.510442:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609816576 : -109714099735040 : ffff9c37320ef600) 00010000:00000001:0.0:1788572470.510444:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.510445:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.510446:0:237083:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.510448:0:237083:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.510449:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.510452:0:237083:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.510455:0:237083:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.510456:0:237083:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.510458:0:237083:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.510460:0:237083:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.510462:0:237083:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372eb6ac00 with handle 0x14415860931683d5 from hash 00010000:00000001:0.0:1788572470.510464:0:237083:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.510465:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.510466:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.510467:0:237083:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.510470:0:237083:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.510472:0:237083:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.510476:0:237083:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.510478:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.510479:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.510480:0:237083:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.510482:0:237083:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.510485:0:237083:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.510490:0:237083:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572470.510495:0:237083:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.510499:0:237083:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.510501:0:237083:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.510503:0:237083:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.510509:0:237083:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6800. 00000100:00000001:0.0:1788572470.510516:0:237083:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.510520:0:237083:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.510524:0:237083:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.510531:0:237083:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.510532:0:237083:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.510536:0:237083:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.510542:0:237083:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.510563:0:237083:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.510591:0:237083:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.510594:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.510596:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.510598:0:237083:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.510600:0:237083:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.510602:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510605:0:237083:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.510611:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.510614:0:237083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931683d5 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xda5:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a20bc expref: -99 pid: 261306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.510621:0:237083:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83740. 00010000:00000001:0.0:1788572470.510630:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.510632:0:237083:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.510633:0:237083:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c372d946f00. 00010000:00000001:0.0:1788572470.510655:0:237083:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.510657:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.510659:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510661:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.510662:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.510663:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510665:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.510667:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.510668:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.510670:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572470.510680:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.510683:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.510685:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.510691:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.510698:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.510700:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.510706:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.510709:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.510711:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.510715:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809935232:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.510720:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.510727:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.510729:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.510732:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.510738:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b6e00. 02000000:00000001:0.0:1788572470.510740:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.510748:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.510753:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c88f0. 00000400:00000010:0.0:1788572470.510761:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572470.510768:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809935232, portal 18 00000100:00000001:0.0:1788572470.510771:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.510772:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000040:0.0:1788572470.510779:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.510786:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.510788:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.510791:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572470.510794:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809935232, offset 0 00000400:00000200:0.0:1788572470.510801:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.510808:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.510816:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.510821:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.510825:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.510828:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.510831:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.510834:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539686:539686:256:4294967295] 192.168.202.148@tcp LPNI seq info [539651:539651:8:4294967295] 00000400:00000200:0.0:1788572470.510839:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.510847:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.510853:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.510859:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572470.510865:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.510883:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.510887:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.510899:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.510904:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15028 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.510907:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.510910:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609533 00000100:00000001:0.0:1788572470.510913:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609650 00000100:00000001:0.0:1788572470.510915:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609533 00000100:00000001:0.0:1788572470.510917:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609650 00000100:00000001:0.0:1788572470.510920:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.510923:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.510926:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.510928:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.510936:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.510938:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572470.510940:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.510941:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.510943:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.510945:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.510960:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.510962:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.511763:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.511769:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0000. 00000400:00000200:0.0:1788572470.511773:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.511779:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.511785:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31dd0 00000400:00000010:0.0:1788572470.511788:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31dd0. 00000100:00000001:0.0:1788572470.511792:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.511796:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.511805:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.511808:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.511815:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.511816:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.511831:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.511833:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.511835:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.511839:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.511846:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.511848:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.511851:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.511853:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.511855:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.511858:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809935360:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.511862:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.511865:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.511868:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.511870:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.511883:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788572470.511885:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.511891:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.511895:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89f8. 00000400:00000010:0.0:1788572470.511899:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572470.511903:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809935360, portal 4 00000100:00000001:0.0:1788572470.511905:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.511906:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788572470.511912:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.511919:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.511921:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.511924:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572470.511926:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809935360, offset 0 00000400:00000200:0.0:1788572470.511931:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.511936:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.511942:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.511945:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.511949:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.511951:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.511954:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.511957:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539687:539687:256:4294967295] 192.168.202.148@tcp LPNI seq info [539652:539652:8:4294967295] 00000400:00000200:0.0:1788572470.511961:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.511967:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.511972:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.511975:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572470.511980:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.511986:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.511989:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.511998:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.512003:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 182090 usecs (183 jiffies) 00000400:00000200:0.0:1788572470.512006:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.512008:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609701 00000100:00000001:0.0:1788572470.512016:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307609650 00000100:00000001:0.0:1788572470.512020:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609701 00000100:00000001:0.0:1788572470.512023:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307609650 00000100:00000001:0.0:1788572470.512026:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.512028:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512031:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512033:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.512039:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.512040:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572470.512042:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.512044:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.512046:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512047:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512058:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.512060:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.512062:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512063:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.512086:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.512089:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0000. 00000400:00000200:0.0:1788572470.512092:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.512096:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.512100:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572470.512102:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572470.512105:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.512108:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.512116:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.512119:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.512125:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.512126:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.514316:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.514334:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.514339:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.514342:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.514351:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.514358:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.514367:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.514372:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.514378:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.514382:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460180 00000400:00000200:0.0:1788572470.514387:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.514393:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529a11 [1] + 224 00000800:00000001:0.0:1788572470.514400:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.514407:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.514411:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.514416:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.514423:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.514431:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.514442:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7480 x1875440809935232/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.514582:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.514592:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.514595:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.516737:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.516750:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.516754:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.516756:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.516763:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.516768:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.516777:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.516781:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.516786:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.516790:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460200 00000400:00000200:0.0:1788572470.516794:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.516800:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529a21 [1] + 224 00000800:00000001:0.0:1788572470.516806:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.516827:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.516830:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.516834:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.516840:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.516845:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.516854:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c6d80 x1875440809935360/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.516941:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.516950:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.516953:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.517506:0:261306:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.517509:0:261306:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.517516:0:261306:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.517519:0:261306:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2eb70 -> ffff9c371ee2eb70 00000020:00000001:0.0:1788572470.517521:0:261306:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2eb70 -> ffff9c371ee2eb70 00000020:00000001:0.0:1788572470.517523:0:261306:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.517526:0:261306:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.517530:0:261306:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.517532:0:261306:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.517534:0:261306:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.517535:0:261306:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.517538:0:261306:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xda5:11031 on OST[1]: s=0 (1) m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.517542:0:261306:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.517543:0:261306:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572464 a=1788572464 c=1788572464 b=0 00020000:00000001:0.0:1788572470.517546:0:261306:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.517548:0:261306:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.517549:0:261306:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2eb70 -> ffff9c371ee2eb70 00000080:00200000:0.0:1788572470.520983:0:261306:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xda5:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.520988:0:261306:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.520992:0:261306:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.520994:0:261306:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3705179720 (ffff9c3711a1d488/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.520998:0:261306:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.521000:0:261306:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3705179720 (ffff9c3711a1d488/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.521004:0:261306:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.521009:0:261306:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.521011:0:261306:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859897800 : -109714849653816 : ffff9c37055c1bc8) 00000020:00000001:0.0:1788572470.521013:0:261306:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.521014:0:261306:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192a4b0 (ffff9c3718da67e8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.521018:0:261306:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.521020:0:261306:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.521023:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.521027:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.521028:0:261306:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.521031:0:261306:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.521033:0:261306:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.521034:0:261306:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.521036:0:261306:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.521037:0:261306:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3705179720 (ffff9c3711a1d488/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.521040:0:261306:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.521041:0:261306:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.521042:0:261306:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192a4b0 (ffff9c3711a1d488/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.521047:0:261306:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313ba20. 00000020:00000001:0.0:1788572470.521055:0:261306:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.521057:0:261306:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192a480. 00020000:00000001:0.0:1788572470.521061:0:261306:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.521062:0:261306:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.521063:0:261306:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.521065:0:261306:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.521068:0:261306:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.521071:0:261306:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.521073:0:261306:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.521074:0:261306:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.521076:0:261306:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.521078:0:261306:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da67e0 00020000:00000001:0.0:1788572470.521083:0:261306:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.521086:0:261306:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.521089:0:261306:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.521092:0:261306:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.521096:0:261306:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.521099:0:261306:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.521101:0:261306:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.521103:0:261306:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.521103:0:261306:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.521105:0:261306:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d480 00000001:00000001:0.0:1788572470.521107:0:261306:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.521140:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03d08 00000080:00000008:0.0:1788572470.521143:0:261306:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-017:[0x200002b17:0xda5:0x0](ffff9c372f0fa1c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.521150:0:261306:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.521152:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03da0 00000080:00002000:0.0:1788572470.521154:0:261306:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-017 (ffff9c370833ca20, parent ffff9c3719d93af8, inode ffff9c372f0fa1c8) hashed, 00000080:00000001:0.0:1788572470.521157:0:261306:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.521199:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.521204:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.521206:0:261306:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.521208:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bc0 00000080:00200000:0.0:1788572470.521210:0:261306:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.521216:0:261306:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572470.521223:0:261306:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.521228:0:261306:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.521231:0:261306:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.521234:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03ae0 00800000:00000002:0.0:1788572470.521236:0:261306:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.521240:0:261306:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.521243:0:261306:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.521245:0:261306:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.521248:0:261306:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.521250:0:261306:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.521252:0:261306:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.521254:0:261306:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.521256:0:261306:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.521259:0:261306:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.521264:0:261306:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.521266:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a039b0 00000002:00010000:0.0:1788572470.521268:0:261306:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.521272:0:261306:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.521274:0:261306:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.521277:0:261306:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.521286:0:261306:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.521289:0:261306:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.521295:0:261306:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.521300:0:261306:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.521306:0:261306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 133 lru_type: 1 02000000:00000001:0.0:1788572470.521319:0:261306:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.521325:0:261306:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.521329:0:261306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 133 lru_type: 1 00000002:00000001:0.0:1788572470.521336:0:261306:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.521338:0:261306:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.521341:0:261306:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.521342:0:261306:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.521344:0:261306:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.521346:0:261306:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.521349:0:261306:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.521351:0:261306:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.521354:0:261306:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6000. 00000080:00000001:0.0:1788572470.521362:0:261306:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.521364:0:261306:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.521367:0:261306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.521371:0:261306:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.521374:0:261306:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.521376:0:261306:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.521378:0:261306:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.521380:0:261306:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.521383:0:261306:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.521385:0:261306:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.521388:0:261306:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.521391:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.521392:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.521393:0:261306:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.521395:0:261306:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.521397:0:261306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a03bf8 00010000:00000001:0.0:1788572470.521399:0:261306:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.521400:0:261306:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.521402:0:261306:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.521403:0:261306:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.521405:0:261306:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.521408:0:261306:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.521412:0:261306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 133 lru_type: 1 00010000:00000001:0.0:1788572470.521418:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.521419:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.521421:0:261306:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.521423:0:261306:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.521425:0:261306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 134 lru_type: 1 00010000:00000001:0.0:1788572470.521434:0:261306:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.521438:0:261306:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.521440:0:261306:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.521444:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.521446:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.521447:0:261306:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.521451:0:261306:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.521452:0:261306:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.521454:0:261306:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.521457:0:261306:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.521459:0:261306:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.521460:0:261306:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.521461:0:261306:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.521464:0:261306:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.521466:0:261306:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.521468:0:261306:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.521471:0:261306:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.521475:0:261306:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.521477:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03b88 00000080:00002000:0.0:1788572470.521478:0:261306:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 17 00000080:00000001:0.0:1788572470.521481:0:261306:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.521483:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bd0 00000080:00200000:0.0:1788572470.521484:0:261306:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.521490:0:261306:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.521494:0:261306:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.521495:0:261306:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.521498:0:261306:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.521501:0:261306:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.521502:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03be8 00000080:00002000:0.0:1788572470.521504:0:261306:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-017(ffff9c370833ca20) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.521506:0:261306:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.521507:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03c30 00000080:00200000:0.0:1788572470.521508:0:261306:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-017, flags=9 00000080:00000001:0.0:1788572470.521523:0:261306:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.521526:0:261306:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.521528:0:261306:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.521529:0:261306:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03e40 00000080:00002000:0.0:1788572470.521530:0:261306:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-017 (ffff9c370833ca20, parent ffff9c3719d93af8, inode ffff9c372f0fa1c8) hashed, 00000080:00000001:0.0:1788572470.521532:0:261306:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.539908:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.539913:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572470.539918:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.539924:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.539930:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1088 00000400:00000010:0.0:1788572470.539932:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1088. 00000100:00000001:0.0:1788572470.539936:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.539940:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809935488/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.539950:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.539953:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72680 x1875440809935488/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.539960:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.539962:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788572470.553422:0:261308:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553426:0:261308:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.553432:0:261308:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.553434:0:261308:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ec38 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572470.553436:0:261308:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ec38 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572470.553438:0:261308:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.553442:0:261308:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.553447:0:261308:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.553449:0:261308:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.553450:0:261308:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.553451:0:261308:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.553454:0:261308:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xda6:11031 on OST[0]: s=0 (1) m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.553457:0:261308:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.553460:0:261308:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.553462:0:261308:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553464:0:261308:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553465:0:261308:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ec38 -> ffff9c371ee2ec38 00000080:00200000:0.0:1788572470.553468:0:261308:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xda6:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.553470:0:261308:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.553472:0:261308:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.553474:0:261308:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff53530 (ffff9c3718da61b8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.553477:0:261308:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.553479:0:261308:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff53530 (ffff9c3718da61b8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.553482:0:261308:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.553486:0:261308:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.553488:0:261308:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293051568 : -109714416500048 : ffff9c371f2d82b0) 00000020:00000001:0.0:1788572470.553489:0:261308:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.553491:0:261308:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cf230 (ffff9c373184dab8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.553494:0:261308:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.553495:0:261308:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.553497:0:261308:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.553498:0:261308:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.553499:0:261308:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.553501:0:261308:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.553503:0:261308:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.553504:0:261308:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.553505:0:261308:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.553507:0:261308:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff53530 (ffff9c3718da61b8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.553510:0:261308:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.553511:0:261308:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.553512:0:261308:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cf230 (ffff9c3718da61b8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.553515:0:261308:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb90a0. 00000020:00000001:0.0:1788572470.553519:0:261308:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.553521:0:261308:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cf200. 00020000:00000001:0.0:1788572470.553525:0:261308:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.553526:0:261308:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.553527:0:261308:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.553529:0:261308:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553532:0:261308:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.553535:0:261308:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.553537:0:261308:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.553538:0:261308:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.553540:0:261308:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.553542:0:261308:0:(cl_object.c:917:cl_env_put()) 1@ffff9c373184dab0 00020000:00000001:0.0:1788572470.553545:0:261308:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.553547:0:261308:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.553558:0:261308:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.553560:0:261308:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.553565:0:261308:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.553567:0:261308:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.553568:0:261308:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.553570:0:261308:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.553571:0:261308:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.553573:0:261308:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000001:00000001:0.0:1788572470.553575:0:261308:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553582:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03d08 00000080:00000008:0.0:1788572470.553585:0:261308:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-018:[0x200002b17:0xda6:0x0](ffff9c372f0f9988), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.553591:0:261308:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.553593:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03da0 00000080:00002000:0.0:1788572470.553595:0:261308:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-018 (ffff9c3719d93000, parent ffff9c3719d93af8, inode ffff9c372f0f9988) hashed, 00000080:00000001:0.0:1788572470.553598:0:261308:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.553630:0:261308:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.553633:0:261308:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.553679:0:261308:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.553680:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bc0 00000080:00200000:0.0:1788572470.553682:0:261308:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.553689:0:261308:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572470.553696:0:261308:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.553699:0:261308:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.553703:0:261308:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.553705:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03ae0 00800000:00000002:0.0:1788572470.553707:0:261308:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.553711:0:261308:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.553714:0:261308:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.553716:0:261308:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.553719:0:261308:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.553721:0:261308:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.553723:0:261308:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.553725:0:261308:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.553727:0:261308:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.553731:0:261308:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.553736:0:261308:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.553737:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a039b0 00000002:00010000:0.0:1788572470.553739:0:261308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.553742:0:261308:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.553744:0:261308:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.553746:0:261308:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.553753:0:261308:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.553756:0:261308:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.553759:0:261308:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.553762:0:261308:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.553766:0:261308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 135 lru_type: 1 02000000:00000001:0.0:1788572470.553773:0:261308:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.553777:0:261308:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.553781:0:261308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 135 lru_type: 1 00000002:00000001:0.0:1788572470.553788:0:261308:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.553790:0:261308:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.553791:0:261308:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.553793:0:261308:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.553795:0:261308:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.553797:0:261308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.553799:0:261308:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.553801:0:261308:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.553805:0:261308:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6600. 00000080:00000001:0.0:1788572470.553811:0:261308:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.553813:0:261308:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.553815:0:261308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.553818:0:261308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.553821:0:261308:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.553824:0:261308:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.553825:0:261308:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.553826:0:261308:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.553828:0:261308:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.553830:0:261308:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.553832:0:261308:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.553834:0:261308:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.553835:0:261308:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.553836:0:261308:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.553837:0:261308:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.553839:0:261308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a03bf8 00010000:00000001:0.0:1788572470.553841:0:261308:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.553842:0:261308:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.553843:0:261308:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.553844:0:261308:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.553846:0:261308:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.553847:0:261308:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.553850:0:261308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 135 lru_type: 1 00010000:00000001:0.0:1788572470.553854:0:261308:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.553856:0:261308:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.553856:0:261308:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.553858:0:261308:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.553859:0:261308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 136 lru_type: 1 00010000:00000001:0.0:1788572470.553865:0:261308:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.553867:0:261308:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.553868:0:261308:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.553870:0:261308:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.553871:0:261308:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.553873:0:261308:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.553884:0:261308:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.553885:0:261308:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.553887:0:261308:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.553889:0:261308:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.553891:0:261308:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.553892:0:261308:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.553893:0:261308:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.553895:0:261308:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.553896:0:261308:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.553898:0:261308:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.553902:0:261308:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.553905:0:261308:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.553908:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03b88 00000080:00002000:0.0:1788572470.553909:0:261308:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 18 00000080:00000001:0.0:1788572470.553912:0:261308:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553914:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bd0 00000080:00200000:0.0:1788572470.553916:0:261308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.553921:0:261308:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.553925:0:261308:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.553926:0:261308:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.553929:0:261308:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.553931:0:261308:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.553932:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03be8 00000080:00002000:0.0:1788572470.553933:0:261308:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-018(ffff9c3719d93000) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.553935:0:261308:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.553937:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03c30 00000080:00200000:0.0:1788572470.553938:0:261308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-018, flags=9 00000080:00000001:0.0:1788572470.553949:0:261308:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.553951:0:261308:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.553954:0:261308:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.553955:0:261308:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03e40 00000080:00002000:0.0:1788572470.553957:0:261308:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-018 (ffff9c3719d93000, parent ffff9c3719d93af8, inode ffff9c372f0f9988) hashed, 00000080:00000001:0.0:1788572470.553960:0:261308:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.570067:0:261310:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.570074:0:261310:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.570077:0:261310:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.570080:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7b20 00000080:00200000:0.0:1788572470.570085:0:261310:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.570092:0:261310:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572470.570099:0:261310:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.570104:0:261310:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.570110:0:261310:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.570112:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7a40 00800000:00000002:0.0:1788572470.570114:0:261310:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.570118:0:261310:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.570121:0:261310:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.570124:0:261310:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.570127:0:261310:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.570130:0:261310:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.570132:0:261310:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.570134:0:261310:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.570136:0:261310:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.570140:0:261310:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.570146:0:261310:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.570147:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7910 00000002:00010000:0.0:1788572470.570149:0:261310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.570153:0:261310:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.570155:0:261310:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.570159:0:261310:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.570167:0:261310:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.570171:0:261310:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.570176:0:261310:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.570180:0:261310:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.570184:0:261310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 136 lru_type: 1 02000000:00000001:0.0:1788572470.570196:0:261310:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.570203:0:261310:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.570207:0:261310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 136 lru_type: 1 00000002:00000001:0.0:1788572470.570213:0:261310:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.570216:0:261310:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.570217:0:261310:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.570219:0:261310:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.570221:0:261310:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.570223:0:261310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.570225:0:261310:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.570228:0:261310:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.570232:0:261310:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6600. 00000080:00000001:0.0:1788572470.570240:0:261310:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.570241:0:261310:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.570244:0:261310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.570247:0:261310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.570250:0:261310:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.570252:0:261310:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.570254:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.570256:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.570260:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.570262:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.570265:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.570267:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.570269:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.570270:0:261310:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.570271:0:261310:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.570274:0:261310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1cb7b58 00010000:00000001:0.0:1788572470.570276:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.570277:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.570278:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.570280:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.570282:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.570285:0:261310:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.570287:0:261310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 136 lru_type: 1 00010000:00000001:0.0:1788572470.570293:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.570294:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.570296:0:261310:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.570298:0:261310:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.570300:0:261310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 137 lru_type: 1 00010000:00000001:0.0:1788572470.570307:0:261310:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.570310:0:261310:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.570313:0:261310:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.570317:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.570318:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.570320:0:261310:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570323:0:261310:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.570325:0:261310:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570327:0:261310:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570330:0:261310:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570332:0:261310:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.570334:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.570335:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.570338:0:261310:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.570339:0:261310:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.570341:0:261310:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.570345:0:261310:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.570349:0:261310:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.570352:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7ae8 00000080:00002000:0.0:1788572470.570353:0:261310:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 18 00000080:00000001:0.0:1788572470.570357:0:261310:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.570359:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7b30 00000080:00200000:0.0:1788572470.570360:0:261310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.570367:0:261310:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.570371:0:261310:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.570373:0:261310:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.570376:0:261310:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.570386:0:261310:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f820. 00000020:00000001:0.0:1788572470.570390:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7ae8 00000080:00200000:0.0:1788572470.570391:0:261310:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-019, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.570394:0:261310:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.570396:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7998 00000080:00200000:0.0:1788572470.570398:0:261310:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-019, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.570404:0:261310:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572470.570406:0:261310:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.570410:0:261310:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.570411:0:261310:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.570413:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb78b8 00800000:00000002:0.0:1788572470.570414:0:261310:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-019' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.570416:0:261310:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.570419:0:261310:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.570421:0:261310:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.570422:0:261310:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.570423:0:261310:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.570425:0:261310:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.570427:0:261310:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.570429:0:261310:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.570431:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7850 00800000:00000002:0.0:1788572470.570433:0:261310:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-019' -> mds #0 00000002:00000001:0.0:1788572470.570436:0:261310:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.570437:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7788 00000002:00010000:0.0:1788572470.570439:0:261310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-019,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.570442:0:261310:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.570444:0:261310:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.570449:0:261310:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572470.570452:0:261310:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.570458:0:261310:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.570462:0:261310:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.570465:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.570466:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.570468:0:261310:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.570470:0:261310:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.570472:0:261310:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.570475:0:261310:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.570482:0:261310:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572470.570489:0:261310:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570494:0:261310:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.570496:0:261310:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570497:0:261310:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.570499:0:261310:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.570501:0:261310:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.570508:0:261310:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572470.570511:0:261310:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.570513:0:261310:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.570517:0:261310:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.570519:0:261310:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.570521:0:261310:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.570524:0:261310:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b600. 00000020:00000001:0.0:1788572470.570528:0:261310:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.570530:0:261310:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b600 with handle 0x14415860931683f1 to hash 00000020:00000001:0.0:1788572470.570532:0:261310:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.570534:0:261310:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.570537:0:261310:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.570540:0:261310:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.570541:0:261310:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.570545:0:261310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.570552:0:261310:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.570561:0:261310:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261310 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.570566:0:261310:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.570569:0:261310:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.570574:0:261310:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572470.570576:0:261310:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572470.570579:0:261310:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.570580:0:261310:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572470.570585:0:261310:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.570589:0:261310:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.570591:0:261310:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.570592:0:261310:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.570603:0:261310:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.570612:0:261310:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.570614:0:261310:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.570617:0:261310:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.570619:0:261310:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.570622:0:261310:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.570626:0:261310:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261310:1875440809935872:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.570630:0:261310:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.570661:0:261310:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.570664:0:261310:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.570666:0:261310:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.570671:0:261310:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9800. 02000000:00000001:0.0:1788572470.570673:0:261310:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.570682:0:261310:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.570686:0:261310:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f20. 00000400:00000010:0.0:1788572470.570694:0:261310:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572470.570701:0:261310:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809935872, portal 10 00000100:00000001:0.0:1788572470.570702:0:261310:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.570704:0:261310:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572470.570710:0:261310:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.570717:0:261310:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.570719:0:261310:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.570723:0:261310:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572470.570725:0:261310:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809935872, offset 0 00000400:00000200:0.0:1788572470.570731:0:261310:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.570739:0:261310:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.570746:0:261310:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.570751:0:261310:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.570755:0:261310:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.570758:0:261310:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.570761:0:261310:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.570764:0:261310:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539691:539691:256:4294967295] 192.168.202.148@tcp LPNI seq info [539656:539656:8:4294967295] 00000400:00000200:0.0:1788572470.570769:0:261310:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.570778:0:261310:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.570784:0:261310:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.570790:0:261310:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572470.570796:0:261310:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.570803:0:261310:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.570808:0:261310:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.570824:0:261310:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.570828:0:261310:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15018 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.570832:0:261310:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.570834:0:261310:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609593 00000100:00000001:0.0:1788572470.570838:0:261310:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609718 00000100:00000001:0.0:1788572470.570840:0:261310:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609593 00000100:00000001:0.0:1788572470.570842:0:261310:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609718 00000100:00000001:0.0:1788572470.570844:0:261310:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.570847:0:261310:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.570849:0:261310:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.570851:0:261310:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.570853:0:261310:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.570856:0:261310:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.570859:0:261310:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.577160:0:261310:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.577246:0:261310:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.577248:0:261310:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.577252:0:261310:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8f20. 00000400:00000200:0.0:1788572470.577257:0:261310:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572470.577260:0:261310:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572470.577263:0:261310:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.577267:0:261310:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.577277:0:261310:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.577283:0:261310:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.577285:0:261310:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.577288:0:261310:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.577292:0:261310:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.577296:0:261310:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.577297:0:261310:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.577301:0:261310:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.577314:0:261310:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=34 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572470.577319:0:261310:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.577321:0:261310:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.577323:0:261310:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.577326:0:261310:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.577331:0:261310:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.577334:0:261310:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.577336:0:261310:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.577339:0:261310:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.577342:0:261310:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.577348:0:261310:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.577354:0:261310:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.577357:0:261310:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.577359:0:261310:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.577362:0:261310:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.577370:0:261310:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261310:1875440809935872:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.577375:0:261310:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.577377:0:261310:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.577379:0:261310:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572470.577382:0:261310:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.577385:0:261310:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.577392:0:261310:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572470.577395:0:261310:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572470.577398:0:261310:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572470.577400:0:261310:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.577403:0:261310:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.577407:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577409:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577412:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=4 00000020:00000001:0.0:1788572470.577414:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577417:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000040:0.0:1788572470.577424:0:261310:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6b600, remote cookie: 0xd9894672d99a2102, flags: 0x1001 00010000:00010000:0.0:1788572470.577429:0:261310:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.577437:0:261310:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xda7:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572470.577440:0:261310:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572470.577447:0:261310:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514e00 count: 3 00010000:00000001:0.0:1788572470.577450:0:261310:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.577453:0:261310:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.577456:0:261310:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.577468:0:261310:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577474:0:261310:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.577478:0:261310:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572470.577480:0:261310:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572470.577482:0:261310:0:(ldlm_lock.c:1024:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572470.577483:0:261310:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572470.577486:0:261310:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xda7:0x0].0x0 (ffff9c372f514e00) refcount = 3 00010000:00000040:0.0:1788572470.577488:0:261310:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.577491:0:261310:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119e00/0x1441586093168350 lrc: 1/0,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f11 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000040:0.0:1788572470.577498:0:261310:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee40600/0x1441586093167dd1 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xda7:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a0c33 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572470.577504:0:261310:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577509:0:261310:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572470.577513:0:261310:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.577514:0:261310:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.577516:0:261310:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.577519:0:261310:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.577522:0:261310:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577528:0:261310:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.577529:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577531:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.577532:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577533:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.577535:0:261310:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.577540:0:261310:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572470.577542:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577543:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577544:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=3 00000020:00000001:0.0:1788572470.577546:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577549:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577858:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577859:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577861:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=4 00000020:00000001:0.0:1788572470.577863:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577866:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577869:0:261310:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.577872:0:261310:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.577885:0:261310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577891:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577892:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.577894:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577895:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577896:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=5 00000020:00000001:0.0:1788572470.577897:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577899:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577901:0:261310:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.577903:0:261310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577908:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577909:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.577911:0:261310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577921:0:261310:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.577923:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577923:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.577924:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577925:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572470.577932:0:261310:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572470.577940:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577941:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577943:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=3 00000020:00000001:0.0:1788572470.577945:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577947:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00000002:00010000:0.0:1788572470.577951:0:261310:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572470.577961:0:261310:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d0916c0. 00010000:00000001:0.0:1788572470.577963:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577964:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.577966:0:261310:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.577968:0:261310:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.577970:0:261310:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572470.577973:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577974:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577975:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=3 00000020:00000001:0.0:1788572470.577976:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577978:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00000002:00010000:0.0:1788572470.577982:0:261310:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.577988:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.577989:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.577991:0:261310:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572470.577993:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.577994:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.577995:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=3 00000020:00000001:0.0:1788572470.577997:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.577999:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000040:0.0:1788572470.578003:0:261310:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514e00 count: 4 00010000:00000001:0.0:1788572470.578005:0:261310:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.578009:0:261310:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.578012:0:261310:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.578015:0:261310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119e00/0x1441586093168350 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f11 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 02000000:00000001:0.0:1788572470.578023:0:261310:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.578028:0:261310:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.578031:0:261310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119e00/0x1441586093168350 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f11 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.578037:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578038:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.578039:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.578041:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.578042:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=3 00000020:00000001:0.0:1788572470.578044:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.578046:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572470.578048:0:261310:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:0.0:1788572470.578051:0:261310:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.578057:0:261310:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.578059:0:261310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.578069:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578070:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.578072:0:261310:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b600/0x14415860931683f1 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xd9894672d99a2102 expref: -99 pid: 261310 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.578078:0:261310:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.578079:0:261310:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578083:0:261310:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:0.0:1788572470.578087:0:261310:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff9c371eeb5a80. 00010000:00000001:0.0:1788572470.578090:0:261310:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:0.0:1788572470.578141:0:261310:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:0.0:1788572470.578144:0:261310:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578146:0:261310:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578147:0:261310:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.578149:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578150:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.578151:0:261310:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572470.578154:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7710 00000002:00002000:0.0:1788572470.578157:0:261310:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-019 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572470.578160:0:261310:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.578163:0:261310:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.578166:0:261310:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572470.578169:0:261310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572470.578172:0:261310:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.578175:0:261310:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.578178:0:261310:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572470.578179:0:261310:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1cb7b20 it_disposition b 00000080:00000001:0.0:1788572470.578182:0:261310:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572470.578187:0:261310:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572470.578190:0:261310:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572470.578192:0:261310:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329063 : 144115373145329063 : 200002b17000da7) 00000080:00000001:0.0:1788572470.578195:0:261310:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572470.578200:0:261310:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.578204:0:261310:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572470.578206:0:261310:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572470.578209:0:261310:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.578212:0:261310:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.578214:0:261310:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.578220:0:261310:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.578223:0:261310:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.578227:0:261310:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572470.578230:0:261310:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572470.578233:0:261310:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xda7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.578237:0:261310:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.578240:0:261310:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20978 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.578243:0:261310:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c372d066ba0) owner ffff9c3704cc27c0 00020000:00000001:0.0:1788572470.578246:0:261310:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.578248:0:261310:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c372d066ba0) owner ffff9c3704cc27c0 00020000:00000010:0.0:1788572470.578251:0:261310:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.578254:0:261310:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572470.578257:0:261310:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572470.578259:0:261310:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xda7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.578261:0:261310:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.578263:0:261310:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572470.578265:0:261310:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572470.578269:0:261310:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329063 : 144115373145329063 : 200002b17000da7) 00000080:00000001:0.0:1788572470.578271:0:261310:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572470.578275:0:261310:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xda7:0x0](ffff9c37087bec08): rc = 0 00000080:00000001:0.0:1788572470.578277:0:261310:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358912314376 : -109714797237240 : ffff9c37087bec08) 00010000:00000001:0.0:1788572470.578279:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.578280:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.578282:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119e00 refcount=3 00000020:00000001:0.0:1788572470.578284:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00010000:00000001:0.0:1788572470.578286:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00000080:00000001:0.0:1788572470.578289:0:261310:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572470.578290:0:261310:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.578291:0:261310:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572470.578293:0:261310:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572470.578294:0:261310:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572470.578295:0:261310:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572470.578296:0:261310:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572470.578298:0:261310:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.578300:0:261310:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.578302:0:261310:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572470.578304:0:261310:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572470.578307:0:261310:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xda7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572470.578310:0:261310:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572470.578313:0:261310:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20978 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572470.578315:0:261310:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c372d066ba0) owner ffff9c3704cc27c0 00020000:00000001:0.0:1788572470.578316:0:261310:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572470.578318:0:261310:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c372d066ba0) owner ffff9c3704cc27c0 00020000:00000010:0.0:1788572470.578319:0:261310:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572470.578321:0:261310:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572470.578323:0:261310:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572470.578324:0:261310:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xda7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572470.578326:0:261310:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.578328:0:261310:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.578330:0:261310:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572470.578333:0:261310:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c372d066ba0) owner 0000000000000000/ffff9c3704cc27c0 00020000:00000002:0.0:1788572470.578337:0:261310:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ecbb4e0 addref 2/0 by ffff9c3704cc27c0. 00020000:00000002:0.0:1788572470.578339:0:261310:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c372d066ba0) owner 0000000000000000/ffff9c3704cc27c0 00020000:00000001:0.0:1788572470.578341:0:261310:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572470.578344:0:261310:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572470.578345:0:261310:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572470.578347:0:261310:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572470.578349:0:261310:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572470.578351:0:261310:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xda7:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572470.578353:0:261310:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572470.578355:0:261310:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578357:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578358:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.578360:0:261310:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572470.578363:0:261310:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572470.578364:0:261310:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.578367:0:261310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xda7:0x0](ffff9c37087bec08) for lock 0x1441586093168350 00800000:00000001:0.0:1788572470.578370:0:261310:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.578372:0:261310:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.578373:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.578374:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.578375:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119e00 refcount=3 00000020:00000001:0.0:1788572470.578377:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00010000:00000001:0.0:1788572470.578379:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00010000:00000001:0.0:1788572470.578381:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578382:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.578384:0:261310:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.578386:0:261310:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572470.578392:0:261310:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c3719d90948 inode ffff9c37087bec08 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572470.578396:0:261310:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572470.578399:0:261310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xda7:0x0](ffff9c37087bec08) 00000080:00010000:0.0:1788572470.578403:0:261310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168350 from it ffffb95bc1cb7b20 00010000:00000001:0.0:1788572470.578404:0:261310:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.578405:0:261310:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.578407:0:261310:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119e00 refcount=3 00000020:00000001:0.0:1788572470.578408:0:261310:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00010000:00000001:0.0:1788572470.578410:0:261310:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291223552 : -109714418328064 : ffff9c371f119e00) 00010000:00000001:0.0:1788572470.578412:0:261310:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.578414:0:261310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119e00/0x1441586093168350 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a1f11 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.578421:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578422:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.578423:0:261310:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.578425:0:261310:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.578428:0:261310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119e00/0x1441586093168350 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f11 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00000001:0.0:1788572470.578435:0:261310:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.578437:0:261310:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.578439:0:261310:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.578443:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578444:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.578446:0:261310:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578447:0:261310:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.578449:0:261310:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578451:0:261310:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578453:0:261310:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.578455:0:261310:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.578456:0:261310:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.578457:0:261310:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.578459:0:261310:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572470.578462:0:261310:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6600. 00000100:00000001:0.0:1788572470.578471:0:261310:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.578476:0:261310:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809935872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.578482:0:261310:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.578484:0:261310:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.578487:0:261310:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109e9800. 02000000:00000001:0.0:1788572470.578490:0:261310:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.578492:0:261310:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.578494:0:261310:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.578496:0:261310:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.578499:0:261310:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572470.578503:0:261310:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.578504:0:261310:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.578506:0:261310:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572470.578509:0:261310:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.578510:0:261310:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.578513:0:261310:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572470.578515:0:261310:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1cb7b20 released 00000080:00000001:0.0:1788572470.578516:0:261310:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572470.578528:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7d08 00000080:00200008:0.0:1788572470.578531:0:261310:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-019:[0x200002b17:0xda7:0x0](ffff9c37087bec08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.578536:0:261310:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.578538:0:261310:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1cb7c80 00000080:00200000:0.0:1788572470.578540:0:261310:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xda7:0x0](ffff9c37087bec08),name=f434.sanity-019 00000080:00000010:0.0:1788572470.578546:0:261310:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572470.578755:0:261310:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000100:00000001:0.0:1788572470.579834:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.579837:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.580199:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.580204:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572470.580209:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.580215:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.580220:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3990 00000400:00000010:0.0:1788572470.580223:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3990. 00000100:00000001:0.0:1788572470.580228:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.580232:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.580241:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.580244:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.580251:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.580253:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.581016:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572470.581019:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572470.581021:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.581028:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.581029:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.581031:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.581033:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.581035:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.581041:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440809936128/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.581048:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.581050:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.581054:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.581056:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.581058:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.581061:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809936128:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.581065:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.581070:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.581072:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.581074:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.581078:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b7e00. 02000000:00000001:0.0:1788572470.581080:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.581087:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.581090:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f20. 00000400:00000010:0.0:1788572470.581096:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572470.581101:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809936128, portal 18 00000100:00000001:0.0:1788572470.581104:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.581105:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788572470.581111:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440809936128/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.581117:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.581120:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.581122:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788572470.581124:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809936128, offset 0 00000400:00000200:0.0:1788572470.581130:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.581136:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.581141:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.581146:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.581149:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.581154:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.581156:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.581159:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539693:539693:256:4294967295] 192.168.202.148@tcp LPNI seq info [539658:539658:8:4294967295] 00000400:00000200:0.0:1788572470.581163:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.581170:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.581175:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.581179:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572470.581184:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.581190:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.581193:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.581206:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.581210:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15015 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.581213:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.581216:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609603 00000100:00000001:0.0:1788572470.581218:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609768 00000100:00000001:0.0:1788572470.581220:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609603 00000100:00000001:0.0:1788572470.581222:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609768 00000100:00000001:0.0:1788572470.581224:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.581226:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.581228:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.581231:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.581244:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.581246:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.581247:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.581249:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.584440:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.584457:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.584461:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.584463:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.584472:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.584480:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.584489:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.584493:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.584498:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.584503:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460500 00000400:00000200:0.0:1788572470.584507:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.584514:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529a69 [1] + 224 00000800:00000001:0.0:1788572470.584521:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.584528:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.584531:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.584535:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.584543:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.584551:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6680 x1875440809936128/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.584562:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c6680 x1875440809936128/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.584587:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.584595:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.584598:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.586094:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.586098:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.586104:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.586106:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.586109:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2000. 00000400:00000200:0.0:1788572470.586113:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572470.586115:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572470.586118:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.586124:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572470.586133:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.586139:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.586141:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.586144:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.586146:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.586148:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.586151:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.586153:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.586161:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=34 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.586165:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=7 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572470.586167:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.586169:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586171:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.586174:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.586177:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.586179:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572470.586181:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.586183:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.586188:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.586195:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.586197:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.586200:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572470.586203:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572470.586205:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586208:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586211:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=4 00000020:00000001:0.0:1788572470.586213:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586217:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586219:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586220:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586222:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=5 00000020:00000001:0.0:1788572470.586223:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586225:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586229:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.586232:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.586236:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586248:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586249:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.586251:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572470.586254:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586255:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586256:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=6 00000020:00000001:0.0:1788572470.586258:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586260:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000040:0.0:1788572470.586266:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204b800, remote cookie: 0xd9894672d99a2109, flags: 0x1001 00010000:00010000:0.0:1788572470.586272:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586280:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572470.586283:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586285:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572470.586288:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572470.586293:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572470.586301:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51f2:0x0].0x0 (ffff9c37039e1600) refcount = 1 00010000:00000040:0.0:1788572470.586304:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572470.586306:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586315:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572470.586317:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.586321:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572470.586323:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.586327:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586333:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572470.586334:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586335:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.586337:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586338:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.586340:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572470.586343:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572470.586347:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586348:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586350:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=5 00000020:00000001:0.0:1788572470.586352:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586354:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00000008:00000001:0.0:1788572470.586358:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572470.586362:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c37053308c0 -> ffff9c37031ffc38 00000008:00010000:0.0:1788572470.586366:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572470.586374:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c37053308c0 -> ffff9c37031ffc38 00000020:00000001:0.0:1788572470.586376:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572470.586379:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572470.586382:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572470.586384:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.586386:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.586388:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c37053308c0 -> ffff9c37031ffc38 00000008:00000001:0.0:1788572470.586390:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572470.586392:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.586394:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572470.586410:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572470.586412:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586415:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586416:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586417:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=6 00000020:00000001:0.0:1788572470.586419:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586421:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586423:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.586426:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586438:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586439:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572470.586442:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586448:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.586449:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586450:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.586451:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586453:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.586454:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.586455:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b800 refcount=5 00000020:00000001:0.0:1788572470.586457:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586459:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609145344 : -109714100406272 : ffff9c373204b800) 00010000:00000001:0.0:1788572470.586460:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.586462:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.586468:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586469:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.586471:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.586473:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.586475:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000daaaeb48/0x14415860931683f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a2109 expref: -99 pid: 261310 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572470.586482:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.586485:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.586487:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.586492:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586493:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.586495:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586498:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.586499:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586501:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586504:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.586506:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.586507:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586508:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.586509:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.586510:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.586512:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.586517:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572470.586525:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809936000:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.586530:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.586533:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.586537:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440809936000/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.586543:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.586546:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.586548:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe95c00. 02000000:00000001:0.0:1788572470.586553:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.586555:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.586557:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572470.586560:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.586562:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711214600. 02000000:00000001:0.0:1788572470.586566:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.586568:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.586569:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788572470.586573:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.586574:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.586576:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.586582:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.586583:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.586586:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.586588:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.603554:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.603560:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572470.603566:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.603571:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.603577:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3aa0 00000400:00000010:0.0:1788572470.603579:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3aa0. 00000100:00000001:0.0:1788572470.603583:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.603587:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6680 x1875440809936256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.603597:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.603600:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6680 x1875440809936256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.603606:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.603608:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572470.610653:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.610658:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572470.610662:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.610667:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.610672:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1d48 00000400:00000010:0.0:1788572470.610674:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1d48. 00000100:00000001:0.0:1788572470.610678:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.610682:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809936384/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.610691:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.610694:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440809936384/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.610701:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.610703:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572470.646997:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.647009:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572470.647018:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.647027:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.647036:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3f68 00000400:00000010:0.0:1788572470.647038:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3f68. 00000100:00000001:0.0:1788572470.647044:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.647051:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5c00 x1875440809936640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.647063:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.647066:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5c00 x1875440809936640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.647084:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.647086:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788572470.649162:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.649165:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649168:0:248770:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.649170:0:248770:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.649173:0:248770:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.649177:0:248770:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.649185:0:248770:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c373204a600 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.649188:0:248770:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.649191:0:248770:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.649198:0:248770:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.649200:0:248770:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.649203:0:248770:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.649209:0:248770:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.649212:0:248770:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.649213:0:248770:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.649215:0:248770:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.649217:0:248770:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204a600 refcount=3 00000020:00000001:0.0:1788572470.649218:0:248770:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609140736 : -109714100410880 : ffff9c373204a600) 00010000:00000001:0.0:1788572470.649221:0:248770:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609140736 : -109714100410880 : ffff9c373204a600) 00010000:00000001:0.0:1788572470.649223:0:248770:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.649226:0:248770:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.649232:0:248770:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.649236:0:248770:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.649238:0:248770:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.649240:0:248770:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xda9:0x0] mode (null) 00000002:00000001:0.0:1788572470.649243:0:248770:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.649245:0:248770:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.649249:0:248770:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25100 count: 4 00010000:00000001:0.0:1788572470.649251:0:248770:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.649254:0:248770:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.649257:0:248770:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.649258:0:248770:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.649262:0:248770:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649265:0:248770:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f118600/0x144158609316835e lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f81 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.649271:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.649272:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.649274:0:248770:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.649276:0:248770:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.649277:0:248770:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.649279:0:248770:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f118600 refcount=3 00000020:00000001:0.0:1788572470.649281:0:248770:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291217408 : -109714418334208 : ffff9c371f118600) 00010000:00000001:0.0:1788572470.649283:0:248770:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291217408 : -109714418334208 : ffff9c371f118600) 00010000:00000001:0.0:1788572470.649285:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.649286:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.649288:0:248770:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.649291:0:248770:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.649292:0:248770:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.649295:0:248770:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.649298:0:248770:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.649299:0:248770:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.649301:0:248770:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.649303:0:248770:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.649306:0:248770:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204a600 with handle 0x144158609316840d from hash 00010000:00000001:0.0:1788572470.649308:0:248770:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.649309:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.649310:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.649311:0:248770:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.649314:0:248770:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.649316:0:248770:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.649320:0:248770:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.649324:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.649325:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.649326:0:248770:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.649329:0:248770:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.649331:0:248770:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.649337:0:248770:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572470.649341:0:248770:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=44 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.649346:0:248770:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.649348:0:248770:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.649352:0:248770:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.649359:0:248770:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7c00. 00000100:00000001:0.0:1788572470.649366:0:248770:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.649369:0:248770:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.649374:0:248770:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.649380:0:248770:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.649383:0:248770:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.649388:0:248770:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.649393:0:248770:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.649403:0:248770:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.649432:0:248770:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.649435:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.649436:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.649438:0:248770:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.649440:0:248770:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.649442:0:248770:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649445:0:248770:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.649452:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.649455:0:248770:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204a600/0x144158609316840d lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xda9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a2148 expref: -99 pid: 261314 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.649463:0:248770:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372eccb2c0. 00010000:00000001:0.0:1788572470.649471:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.649473:0:248770:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.649475:0:248770:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c37380b6000. 00010000:00000001:0.0:1788572470.649478:0:248770:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.649480:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.649481:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649483:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.649484:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.649485:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649487:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.649489:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.649490:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.649491:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572470.649500:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572470.649504:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572470.649506:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.649514:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.649517:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.649519:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.649521:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.649523:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.649529:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.649542:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.649544:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.649548:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.649564:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.649567:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.649571:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809936768:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.649576:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.649582:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.649585:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.649588:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.649594:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572470.649596:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.649603:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.649607:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c82c0. 00000400:00000010:0.0:1788572470.649614:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572470.649622:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809936768, portal 18 00000100:00000001:0.0:1788572470.649624:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.649626:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572470.650169:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.650179:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.650183:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.650186:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572470.650190:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809936768, offset 0 00000400:00000200:0.0:1788572470.650196:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.650205:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.650214:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.650219:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.650223:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.650227:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.650229:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.650234:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539698:539698:256:4294967295] 192.168.202.148@tcp LPNI seq info [539663:539663:8:4294967295] 00000400:00000200:0.0:1788572470.650238:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.650247:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.650253:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.650308:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572470.650315:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.650324:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.650329:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.650342:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.650348:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15002 usecs (16 jiffies) 00000400:00000200:0.0:1788572470.650351:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.650354:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609672 00000100:00000001:0.0:1788572470.650357:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609786 00000100:00000001:0.0:1788572470.650359:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609672 00000100:00000001:0.0:1788572470.650361:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609786 00000100:00000001:0.0:1788572470.650363:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.650366:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.650369:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.650373:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.653201:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.653214:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788572470.653225:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.653235:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.653243:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31d48 00000400:00000010:0.0:1788572470.653246:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31d48. 00000100:00000001:0.0:1788572470.653252:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.653260:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.653274:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.653277:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.653286:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.653288:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.655912:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.655928:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.655931:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.655934:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.655943:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.655950:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.655959:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.655964:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.655969:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.655974:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460780 00000400:00000200:0.0:1788572470.655978:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.655985:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529aa9 [1] + 224 00000800:00000001:0.0:1788572470.655993:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.656001:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.656004:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.656008:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.656015:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.656024:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.656033:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.656048:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.656056:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.656059:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.656070:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.656074:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.656152:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.656154:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.656157:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c82c0. 00000400:00000200:0.0:1788572470.656161:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788572470.656163:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788572470.656166:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.656170:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572470.656179:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.656184:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.656186:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.656189:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.656192:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.656194:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.656196:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.656199:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.656208:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce538 time=23 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.656212:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.656215:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.656217:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.656220:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.656224:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.656228:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.656230:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.656232:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.656234:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.656239:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.656246:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.656249:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.656251:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.656255:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572470.656262:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809936768:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.656267:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.656270:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.656273:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809936768/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.656279:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.656281:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.656283:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572470.656289:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.656291:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.656293:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.656295:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.656297:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7c00. 02000000:00000001:0.0:1788572470.656301:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.656303:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.656305:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572470.656307:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.656309:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.656311:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.656317:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.656319:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.656321:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.656322:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.661057:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.661066:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572470.661087:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.661095:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.661102:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1f68 00000400:00000010:0.0:1788572470.661105:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1f68. 00000100:00000001:0.0:1788572470.661111:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.661118:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b100 x1875440809936896/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.661131:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.661134:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440809936896/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.661141:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.661145:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.661452:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.661463:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.661466:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.661469:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.661475:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.661481:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.661488:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.661492:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.661497:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.661501:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460800 00000400:00000200:0.0:1788572470.661506:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.661511:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ab9 [1] + 224 00000800:00000001:0.0:1788572470.661517:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.661524:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.661527:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.661530:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.661535:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.661539:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b100 x1875440809936896/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.661549:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6b100 x1875440809936896/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.661791:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.661798:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.661802:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.697446:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.697455:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572470.697466:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.697474:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.697482:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1198 00000400:00000010:0.0:1788572470.697485:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1198. 00000100:00000001:0.0:1788572470.697491:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.697500:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809937024/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.697518:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.697522:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72680 x1875440809937024/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.697530:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.697532:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.700404:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.700422:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.700427:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.700429:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.700439:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.700446:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.700455:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.700460:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.700465:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.700469:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417460880 00000400:00000200:0.0:1788572470.700474:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.700480:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ac1 [1] + 224 00000800:00000001:0.0:1788572470.700487:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.700494:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.700497:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.700501:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.700508:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.700518:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809937024/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.700529:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec72680 x1875440809937024/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.701279:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.701293:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.701297:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.703738:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.703742:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.703746:0:237083:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.703749:0:237083:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.703751:0:237083:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.703755:0:237083:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.703763:0:237083:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c371f010000 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.703766:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.703769:0:237083:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.703776:0:237083:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.703778:0:237083:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.703780:0:237083:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.703786:0:237083:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.703789:0:237083:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.703790:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.703791:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.703793:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f010000 refcount=3 00000020:00000001:0.0:1788572470.703795:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359290134528 : -109714419417088 : ffff9c371f010000) 00010000:00000001:0.0:1788572470.703797:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359290134528 : -109714419417088 : ffff9c371f010000) 00010000:00000001:0.0:1788572470.703800:0:237083:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.703802:0:237083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.703884:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.703889:0:237083:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.703891:0:237083:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.703893:0:237083:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xdaa:0x0] mode (null) 00000002:00000001:0.0:1788572470.703896:0:237083:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.703898:0:237083:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.703901:0:237083:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7900 count: 4 00010000:00000001:0.0:1788572470.703904:0:237083:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.703906:0:237083:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.703908:0:237083:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.703910:0:237083:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.703913:0:237083:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.703916:0:237083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f118200/0x1441586093168365 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f9d expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00000001:0.0:1788572470.703922:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.703923:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.703924:0:237083:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.703926:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.703927:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.703928:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f118200 refcount=2 00000020:00000001:0.0:1788572470.703930:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291216384 : -109714418335232 : ffff9c371f118200) 00010000:00000001:0.0:1788572470.703932:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291216384 : -109714418335232 : ffff9c371f118200) 00010000:00000001:0.0:1788572470.703934:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.703935:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.703936:0:237083:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.703940:0:237083:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.703941:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.703944:0:237083:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.703948:0:237083:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.703949:0:237083:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.703951:0:237083:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.703953:0:237083:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.703955:0:237083:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f010000 with handle 0x144158609316841b from hash 00010000:00000001:0.0:1788572470.703958:0:237083:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.703959:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.703960:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.703961:0:237083:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.703964:0:237083:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.703966:0:237083:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.703970:0:237083:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.703973:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.703974:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.703975:0:237083:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.703977:0:237083:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.703980:0:237083:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.703985:0:237083:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572470.703988:0:237083:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.703990:0:237083:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.703992:0:237083:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.703994:0:237083:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.704000:0:237083:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6400. 00000100:00000001:0.0:1788572470.704007:0:237083:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.704010:0:237083:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.704014:0:237083:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.704019:0:237083:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.704021:0:237083:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.704024:0:237083:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.704029:0:237083:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.704038:0:237083:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6680 x1875440809937152/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.705043:0:237083:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.705047:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.705048:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.705050:0:237083:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.705053:0:237083:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.705055:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.705059:0:237083:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.705075:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.705078:0:237083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010000/0x144158609316841b lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xdaa:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a216b expref: -99 pid: 261316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.705091:0:237083:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c37079851c0. 00010000:00000001:0.0:1788572470.705100:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.705101:0:237083:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.705103:0:237083:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c3711a9e840. 00010000:00000001:0.0:1788572470.705106:0:237083:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.705109:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.705110:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.705113:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.705114:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.705115:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.705117:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.705118:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.705119:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.705121:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:0.0:1788572470.709740:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709753:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.709756:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709759:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.709766:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709771:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.709778:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709782:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709786:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709790:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460900 00000400:00000200:0.0:1788572470.709795:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709801:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529ac9 [1] + 224 00000800:00000001:0.0:1788572470.709808:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709820:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.709823:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.709827:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.709831:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.709837:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6680 x1875440809937152/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.709848:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6680 x1875440809937152/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.709858:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.709865:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709868:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.709870:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709872:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.709884:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709888:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.709893:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709895:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709897:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709900:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460980 00000400:00000200:0.0:1788572470.709903:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.709908:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ad9 [1] + 224 00000800:00000001:0.0:1788572470.709912:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709918:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.709920:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.709923:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.709926:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.709930:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809937280/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.709938:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec72680 x1875440809937280/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.709945:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.709948:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.709952:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.711318:0:261316:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711322:0:261316:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.711328:0:261316:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.711331:0:261316:0:(cl_object.c:143:cl_object_top()) ffff9c3732162148 -> ffff9c3732162148 00000020:00000001:0.0:1788572470.711334:0:261316:0:(cl_object.c:143:cl_object_top()) ffff9c3732162148 -> ffff9c3732162148 00000020:00000001:0.0:1788572470.711335:0:261316:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.711340:0:261316:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.711344:0:261316:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.711346:0:261316:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.711348:0:261316:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.711349:0:261316:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.711351:0:261316:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdaa:11031 on OST[0]: s=0 (1) m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.711355:0:261316:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.711357:0:261316:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.711360:0:261316:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711362:0:261316:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711363:0:261316:0:(cl_object.c:143:cl_object_top()) ffff9c3732162148 -> ffff9c3732162148 00000080:00200000:0.0:1788572470.711366:0:261316:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdaa:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.711368:0:261316:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.711371:0:261316:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.711373:0:261316:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff53c20 (ffff9c3718da6638/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.711377:0:261316:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.711378:0:261316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff53c20 (ffff9c3718da6638/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.711381:0:261316:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.711385:0:261316:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.711386:0:261316:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293057992 : -109714416493624 : ffff9c371f2d9bc8) 00000020:00000001:0.0:1788572470.711388:0:261316:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.711390:0:261316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cfe30 (ffff9c3718da6e18/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.711392:0:261316:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.711394:0:261316:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.711395:0:261316:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.711397:0:261316:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.711398:0:261316:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.711400:0:261316:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.711402:0:261316:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.711403:0:261316:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.711404:0:261316:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.711406:0:261316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff53c20 (ffff9c3718da6638/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.711409:0:261316:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.711410:0:261316:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.711411:0:261316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cfe30 (ffff9c3718da6638/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.711414:0:261316:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb8ab0. 00000020:00000001:0.0:1788572470.711418:0:261316:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.711420:0:261316:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cfe00. 00020000:00000001:0.0:1788572470.711423:0:261316:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.711424:0:261316:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.711425:0:261316:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.711427:0:261316:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711430:0:261316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.711432:0:261316:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.711434:0:261316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.711435:0:261316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.711437:0:261316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.711439:0:261316:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6e10 00020000:00000001:0.0:1788572470.711442:0:261316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.711444:0:261316:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.711446:0:261316:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.711448:0:261316:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdaa:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.711452:0:261316:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.711453:0:261316:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.711455:0:261316:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.711457:0:261316:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.711458:0:261316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.711459:0:261316:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6630 00000001:00000001:0.0:1788572470.711461:0:261316:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711469:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03d08 00000080:00000008:0.0:1788572470.711472:0:261316:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-022:[0x200002b17:0xdaa:0x0](ffff9c37081fa1c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.711479:0:261316:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.711480:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03da0 00000080:00002000:0.0:1788572470.711483:0:261316:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-022 (ffff9c3719d93870, parent ffff9c3719d93af8, inode ffff9c37081fa1c8) hashed, 00000080:00000001:0.0:1788572470.711486:0:261316:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.711518:0:261316:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.711522:0:261316:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.711523:0:261316:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.711525:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bc0 00000080:00200000:0.0:1788572470.711527:0:261316:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.711535:0:261316:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572470.711541:0:261316:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.711546:0:261316:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.711549:0:261316:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.711622:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03ae0 00800000:00000002:0.0:1788572470.711624:0:261316:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.711628:0:261316:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.711631:0:261316:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.711633:0:261316:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.711660:0:261316:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.711663:0:261316:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.711665:0:261316:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.711667:0:261316:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.711669:0:261316:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.711672:0:261316:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.711678:0:261316:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.711680:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a039b0 00000002:00010000:0.0:1788572470.711682:0:261316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.711685:0:261316:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.711687:0:261316:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.711689:0:261316:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.711697:0:261316:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.711699:0:261316:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.711704:0:261316:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.711707:0:261316:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.711711:0:261316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 143 lru_type: 1 02000000:00000001:0.0:1788572470.711720:0:261316:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.711725:0:261316:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.711728:0:261316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 143 lru_type: 1 00000002:00000001:0.0:1788572470.711734:0:261316:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.711736:0:261316:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.711738:0:261316:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.711740:0:261316:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.711742:0:261316:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.711744:0:261316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.711747:0:261316:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.711751:0:261316:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.711754:0:261316:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6400. 00000080:00000001:0.0:1788572470.711759:0:261316:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.711760:0:261316:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.711763:0:261316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.711766:0:261316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.711770:0:261316:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.711772:0:261316:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.711773:0:261316:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.711775:0:261316:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.711777:0:261316:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.711779:0:261316:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.711781:0:261316:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.711783:0:261316:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.711784:0:261316:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.711786:0:261316:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.711787:0:261316:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.711790:0:261316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1a03bf8 00010000:00000001:0.0:1788572470.711791:0:261316:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.711792:0:261316:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.711793:0:261316:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.711795:0:261316:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.711796:0:261316:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.711798:0:261316:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.711801:0:261316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 143 lru_type: 1 00010000:00000001:0.0:1788572470.711806:0:261316:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.711807:0:261316:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.711808:0:261316:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.711888:0:261316:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.711891:0:261316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 144 lru_type: 1 00010000:00000001:0.0:1788572470.711898:0:261316:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.711900:0:261316:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.711901:0:261316:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.711903:0:261316:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.711904:0:261316:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.711906:0:261316:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.711908:0:261316:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.711909:0:261316:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.711911:0:261316:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.711913:0:261316:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.711914:0:261316:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.711915:0:261316:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.711916:0:261316:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.711919:0:261316:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.711920:0:261316:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.711922:0:261316:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.711926:0:261316:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.711929:0:261316:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.711931:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03b88 00000080:00002000:0.0:1788572470.711932:0:261316:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 22 00000080:00000001:0.0:1788572470.711935:0:261316:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711937:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03bd0 00000080:00200000:0.0:1788572470.711938:0:261316:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.711943:0:261316:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.711947:0:261316:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.711948:0:261316:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.711951:0:261316:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.711953:0:261316:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.711954:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03be8 00000080:00002000:0.0:1788572470.711956:0:261316:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-022(ffff9c3719d93870) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.711957:0:261316:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.711958:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03c30 00000080:00200000:0.0:1788572470.711960:0:261316:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-022, flags=9 00000080:00000001:0.0:1788572470.711971:0:261316:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.711974:0:261316:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.711977:0:261316:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.711978:0:261316:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1a03e40 00000080:00002000:0.0:1788572470.711979:0:261316:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-022 (ffff9c3719d93870, parent ffff9c3719d93af8, inode ffff9c37081fa1c8) hashed, 00000080:00000001:0.0:1788572470.711983:0:261316:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.729357:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.729364:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.729366:0:261318:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.729370:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fb20 00000080:00200000:0.0:1788572470.729374:0:261318:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.729381:0:261318:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572470.729387:0:261318:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.729391:0:261318:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.729395:0:261318:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.729398:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fa40 00800000:00000002:0.0:1788572470.729400:0:261318:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.729404:0:261318:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.729407:0:261318:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.729409:0:261318:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.729411:0:261318:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.729413:0:261318:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.729416:0:261318:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.729417:0:261318:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.729420:0:261318:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.729423:0:261318:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.729428:0:261318:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.729429:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f910 00000002:00010000:0.0:1788572470.729431:0:261318:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.729435:0:261318:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.729437:0:261318:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.729439:0:261318:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.729446:0:261318:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.729449:0:261318:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.729455:0:261318:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.729459:0:261318:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.729465:0:261318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 144 lru_type: 1 02000000:00000001:0.0:1788572470.729477:0:261318:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.729481:0:261318:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.729485:0:261318:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 144 lru_type: 1 00000002:00000001:0.0:1788572470.729492:0:261318:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.729495:0:261318:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.729496:0:261318:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.729498:0:261318:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.729500:0:261318:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.729502:0:261318:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.729504:0:261318:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.729507:0:261318:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.729510:0:261318:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6400. 00000080:00000001:0.0:1788572470.729517:0:261318:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.729519:0:261318:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.729522:0:261318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.729525:0:261318:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.729528:0:261318:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.729529:0:261318:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.729531:0:261318:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.729533:0:261318:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.729536:0:261318:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.729538:0:261318:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.729541:0:261318:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.729544:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.729545:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.729546:0:261318:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.729548:0:261318:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.729550:0:261318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d4fb58 00010000:00000001:0.0:1788572470.729553:0:261318:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.729554:0:261318:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.729555:0:261318:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.729557:0:261318:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.729559:0:261318:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.729561:0:261318:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.729564:0:261318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 144 lru_type: 1 00010000:00000001:0.0:1788572470.729570:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.729571:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.729573:0:261318:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.729575:0:261318:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.729578:0:261318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 145 lru_type: 1 00010000:00000001:0.0:1788572470.729584:0:261318:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.729587:0:261318:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.729589:0:261318:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.729593:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.729594:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.729595:0:261318:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729598:0:261318:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.729600:0:261318:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729602:0:261318:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729604:0:261318:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729606:0:261318:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.729607:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.729608:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.729611:0:261318:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.729613:0:261318:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.729615:0:261318:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.729619:0:261318:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.729622:0:261318:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.729625:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fae8 00000080:00002000:0.0:1788572470.729627:0:261318:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 22 00000080:00000001:0.0:1788572470.729629:0:261318:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.729631:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fb30 00000080:00200000:0.0:1788572470.729664:0:261318:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.729669:0:261318:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.729673:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.729675:0:261318:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.729678:0:261318:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.729685:0:261318:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310ff00. 00000020:00000001:0.0:1788572470.729688:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fae8 00000080:00200000:0.0:1788572470.729691:0:261318:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-023, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.729694:0:261318:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.729695:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f998 00000080:00200000:0.0:1788572470.729697:0:261318:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-023, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.729703:0:261318:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572470.729705:0:261318:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.729708:0:261318:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.729710:0:261318:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.729711:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f8b8 00800000:00000002:0.0:1788572470.729713:0:261318:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-023' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.729715:0:261318:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.729718:0:261318:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.729719:0:261318:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.729721:0:261318:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.729722:0:261318:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.729724:0:261318:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.729725:0:261318:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.729727:0:261318:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.729729:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f850 00800000:00000002:0.0:1788572470.729731:0:261318:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-023' -> mds #0 00000002:00000001:0.0:1788572470.729734:0:261318:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.729735:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f788 00000002:00010000:0.0:1788572470.729737:0:261318:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-023,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.729740:0:261318:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.729743:0:261318:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.729748:0:261318:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572470.729752:0:261318:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.729757:0:261318:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.729762:0:261318:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.729764:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.729765:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.729767:0:261318:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.729769:0:261318:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.729771:0:261318:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.729774:0:261318:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.729781:0:261318:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572470.729787:0:261318:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729792:0:261318:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.729794:0:261318:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729796:0:261318:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.729797:0:261318:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.729799:0:261318:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.729805:0:261318:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572470.729961:0:261318:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.729964:0:261318:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.729969:0:261318:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.729971:0:261318:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.729973:0:261318:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.729978:0:261318:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6be00. 00000020:00000001:0.0:1788572470.729981:0:261318:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.729983:0:261318:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6be00 with handle 0x1441586093168429 to hash 00000020:00000001:0.0:1788572470.729985:0:261318:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.729987:0:261318:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572470.729990:0:261318:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572470.729993:0:261318:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.729995:0:261318:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.729998:0:261318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168429 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261318 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.730010:0:261318:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168429 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261318 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.730021:0:261318:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6be00/0x1441586093168429 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261318 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.730026:0:261318:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.730028:0:261318:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.730034:0:261318:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.730036:0:261318:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572470.730039:0:261318:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.730041:0:261318:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572470.730045:0:261318:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.730049:0:261318:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.730051:0:261318:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.730053:0:261318:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.730064:0:261318:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809937408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.730137:0:261318:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.730139:0:261318:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.730143:0:261318:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.730146:0:261318:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.730148:0:261318:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.730153:0:261318:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261318:1875440809937408:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.730158:0:261318:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.730164:0:261318:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.730166:0:261318:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.730168:0:261318:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.730173:0:261318:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9800. 02000000:00000001:0.0:1788572470.730175:0:261318:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.730183:0:261318:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.730187:0:261318:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c82c0. 00000400:00000010:0.0:1788572470.730196:0:261318:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572470.730203:0:261318:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809937408, portal 10 00000100:00000001:0.0:1788572470.730205:0:261318:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.730206:0:261318:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572470.730212:0:261318:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809937408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.730220:0:261318:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.730223:0:261318:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.730225:0:261318:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572470.730228:0:261318:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809937408, offset 0 00000400:00000200:0.0:1788572470.730235:0:261318:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.730243:0:261318:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.730250:0:261318:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.730254:0:261318:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.730258:0:261318:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.730262:0:261318:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.730265:0:261318:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.730269:0:261318:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539703:539703:256:4294967295] 192.168.202.148@tcp LPNI seq info [539668:539668:8:4294967295] 00000400:00000200:0.0:1788572470.730273:0:261318:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.730282:0:261318:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.730288:0:261318:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.730293:0:261318:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572470.730299:0:261318:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.730307:0:261318:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.730311:0:261318:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.730320:0:261318:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.730324:0:261318:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14995 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.730327:0:261318:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.730330:0:261318:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609751 00000100:00000001:0.0:1788572470.730333:0:261318:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609882 00000100:00000001:0.0:1788572470.730336:0:261318:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609751 00000100:00000001:0.0:1788572470.730339:0:261318:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609882 00000100:00000001:0.0:1788572470.730341:0:261318:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.730344:0:261318:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.730346:0:261318:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.730348:0:261318:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.730351:0:261318:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.730353:0:261318:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.730355:0:261318:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.739265:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.739283:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.739287:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.739291:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.739301:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.739308:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.739318:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.739322:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.739327:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.739331:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417460a00 00000400:00000200:0.0:1788572470.739336:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.739342:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ae1 [1] + 224 00000800:00000001:0.0:1788572470.739350:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.739357:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.739360:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.739365:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.739373:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.739383:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6680 x1875440809937408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.739394:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b6680 x1875440809937408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.739420:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.739428:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.739432:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.744361:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.744367:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572470.744372:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.744378:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.744384:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1000 00000400:00000010:0.0:1788572470.744386:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1000. 00000100:00000001:0.0:1788572470.744390:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.744396:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68700 x1875440809937536/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.744406:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.744410:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440809937536/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.744417:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.744419:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572470.747825:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.747837:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572470.747846:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.747854:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.747862:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788572470.747864:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788572470.747870:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.747886:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7100 x1875440809937664/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.747898:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.747902:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440809937664/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.747908:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.747910:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.751424:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.751433:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.751435:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.751438:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.751444:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.751450:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.751459:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.751463:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.751467:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.751471:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460b00 00000400:00000200:0.0:1788572470.751476:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.751481:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529af9 [1] + 224 00000800:00000001:0.0:1788572470.751487:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.751958:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.751962:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.751968:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.751974:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.751981:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7100 x1875440809937664/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.751990:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c7100 x1875440809937664/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.752008:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:0.0:1788572470.752993:0:261318:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.752998:0:261318:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.753005:0:261318:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.753008:0:261318:0:(cl_object.c:143:cl_object_top()) ffff9c3732162530 -> ffff9c3732162530 00000020:00000001:0.0:1788572470.753010:0:261318:0:(cl_object.c:143:cl_object_top()) ffff9c3732162530 -> ffff9c3732162530 00000020:00000001:0.0:1788572470.753012:0:261318:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.753016:0:261318:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.753021:0:261318:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.753023:0:261318:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.753025:0:261318:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.753026:0:261318:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.753028:0:261318:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdab:11031 on OST[1]: s=0 (1) m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.753031:0:261318:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.753033:0:261318:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.753037:0:261318:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753038:0:261318:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753040:0:261318:0:(cl_object.c:143:cl_object_top()) ffff9c3732162530 -> ffff9c3732162530 00000080:00200000:0.0:1788572470.753042:0:261318:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdab:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.753045:0:261318:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.753047:0:261318:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.753050:0:261318:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.753053:0:261318:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.753054:0:261318:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.753058:0:261318:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.753061:0:261318:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.753063:0:261318:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572470.753065:0:261318:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.753066:0:261318:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192aeb0 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.753138:0:261318:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.753140:0:261318:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.753141:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.753143:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.753144:0:261318:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.753146:0:261318:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.753147:0:261318:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.753148:0:261318:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.753149:0:261318:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.753151:0:261318:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.753154:0:261318:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.753156:0:261318:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.753157:0:261318:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192aeb0 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.753160:0:261318:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313b1d0. 00000020:00000001:0.0:1788572470.753164:0:261318:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.753165:0:261318:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192ae80. 00020000:00000001:0.0:1788572470.753168:0:261318:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.753169:0:261318:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.753170:0:261318:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.753172:0:261318:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753175:0:261318:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.753178:0:261318:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.753180:0:261318:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.753181:0:261318:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.753184:0:261318:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.753186:0:261318:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572470.753189:0:261318:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.753191:0:261318:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.753193:0:261318:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.753196:0:261318:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdab:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.753200:0:261318:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.753203:0:261318:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.753205:0:261318:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.753207:0:261318:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.753208:0:261318:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.753209:0:261318:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572470.753211:0:261318:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753220:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fd08 00000080:00000008:0.0:1788572470.753223:0:261318:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-023:[0x200002b17:0xdab:0x0](ffff9c37081f9148), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.753229:0:261318:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.753231:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fda0 00000080:00002000:0.0:1788572470.753233:0:261318:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-023 (ffff9c3719f64ca8, parent ffff9c3719d93af8, inode ffff9c37081f9148) hashed, 00000080:00000001:0.0:1788572470.753236:0:261318:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.753273:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.753277:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.753279:0:261318:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.753281:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbc0 00000080:00200000:0.0:1788572470.753283:0:261318:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.753289:0:261318:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572470.753295:0:261318:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.753299:0:261318:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.753302:0:261318:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.753305:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fae0 00800000:00000002:0.0:1788572470.753307:0:261318:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.753310:0:261318:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.753313:0:261318:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.753316:0:261318:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.753318:0:261318:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.753321:0:261318:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.753323:0:261318:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.753325:0:261318:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.753327:0:261318:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.753331:0:261318:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.753336:0:261318:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.753338:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f9b0 00000002:00010000:0.0:1788572470.753340:0:261318:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.753343:0:261318:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.753345:0:261318:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.753347:0:261318:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.753354:0:261318:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.753356:0:261318:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.753360:0:261318:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.753363:0:261318:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.753367:0:261318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 145 lru_type: 1 02000000:00000001:0.0:1788572470.753376:0:261318:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.753380:0:261318:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.753383:0:261318:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 145 lru_type: 1 00000002:00000001:0.0:1788572470.753391:0:261318:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.753394:0:261318:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.753395:0:261318:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.753398:0:261318:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.753399:0:261318:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.753401:0:261318:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.753404:0:261318:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.753407:0:261318:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.753409:0:261318:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572470.753415:0:261318:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.753417:0:261318:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.753420:0:261318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.753423:0:261318:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.753427:0:261318:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.753429:0:261318:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.753430:0:261318:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.753432:0:261318:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.753434:0:261318:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.753436:0:261318:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.753438:0:261318:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.753441:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.753442:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.753443:0:261318:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.753444:0:261318:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.753447:0:261318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d4fbf8 00010000:00000001:0.0:1788572470.753448:0:261318:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.753449:0:261318:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.753450:0:261318:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.753452:0:261318:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.753454:0:261318:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.753456:0:261318:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.753458:0:261318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 145 lru_type: 1 00010000:00000001:0.0:1788572470.753464:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.753465:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.753466:0:261318:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.753468:0:261318:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.753470:0:261318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 146 lru_type: 1 00010000:00000001:0.0:1788572470.753476:0:261318:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.753478:0:261318:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.753479:0:261318:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.753482:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.753483:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.753484:0:261318:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.753486:0:261318:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.753488:0:261318:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.753490:0:261318:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.753492:0:261318:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.753494:0:261318:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.753495:0:261318:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.753496:0:261318:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.753498:0:261318:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.753500:0:261318:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.753502:0:261318:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.753506:0:261318:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.753509:0:261318:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.753511:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fb88 00000080:00002000:0.0:1788572470.753513:0:261318:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 23 00000080:00000001:0.0:1788572470.753515:0:261318:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753517:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbd0 00000080:00200000:0.0:1788572470.753518:0:261318:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.753523:0:261318:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.753526:0:261318:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.753528:0:261318:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.753530:0:261318:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.753532:0:261318:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.753533:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbe8 00000080:00002000:0.0:1788572470.753534:0:261318:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-023(ffff9c3719f64ca8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.753536:0:261318:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.753538:0:261318:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fc30 00000080:00200000:0.0:1788572470.753539:0:261318:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-023, flags=9 00000080:00000001:0.0:1788572470.753549:0:261318:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.754184:0:261318:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000001:00000001:0.0:1788572470.783829:0:261320:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572470.783832:0:261320:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.783834:0:261320:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572470.783837:0:261320:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572470.783842:0:261320:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e210 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572470.783844:0:261320:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572470.783846:0:261320:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572470.783848:0:261320:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdac:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572470.783851:0:261320:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.783853:0:261320:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572470.783855:0:261320:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.783856:0:261320:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.783858:0:261320:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.783861:0:261320:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdac:0x0]io ffff9c371ff50bf0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572470.783864:0:261320:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711ace9b0) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572470.783867:0:261320:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572470.783868:0:261320:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572470.783871:0:261320:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572470.783872:0:261320:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.783885:0:261320:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572470.783887:0:261320:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572470.783889:0:261320:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.783890:0:261320:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.783892:0:261320:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572470.783893:0:261320:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572470.783894:0:261320:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711ace9b0) owner 0000000000000000/ffff9c372d1c4f80 00000020:00000001:0.0:1788572470.783896:0:261320:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572470.783898:0:261320:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572470.783900:0:261320:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdac:0x0] 00000020:00000001:0.0:1788572470.783902:0:261320:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572470.783903:0:261320:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572470.783906:0:261320:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572470.783907:0:261320:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572470.783912:0:261320:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572470.783918:0:261320:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676100. 00020000:00000002:0.0:1788572470.783920:0:261320:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572470.783922:0:261320:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572470.783925:0:261320:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.783926:0:261320:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572470.783927:0:261320:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572470.783929:0:261320:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572470.783931:0:261320:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572470.783935:0:261320:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e148 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572470.783936:0:261320:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572470.783940:0:261320:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572470.783941:0:261320:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572470.783942:0:261320:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.783944:0:261320:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.783946:0:261320:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572470.783948:0:261320:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572470.783951:0:261320:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6000. 00000008:00010000:0.0:1788572470.783955:0:261320:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572470.783957:0:261320:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572470.783958:0:261320:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572470.783960:0:261320:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676130, osc lock ffff9c37030a6000, flags 0x400001002 00000020:00000001:0.0:1788572470.783962:0:261320:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.783963:0:261320:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.783965:0:261320:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725056 : -109714177826560 : ffff9c372d676100) 00000020:00000001:0.0:1788572470.783967:0:261320:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572470.783968:0:261320:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572470.783969:0:261320:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.783971:0:261320:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.783973:0:261320:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572470.783975:0:261320:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572470.783976:0:261320:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572470.783978:0:261320:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572470.783979:0:261320:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.783980:0:261320:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572470.783981:0:261320:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572470.783983:0:261320:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572470.783985:0:261320:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572470.783988:0:261320:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572470.783990:0:261320:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572470.783991:0:261320:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572470.783993:0:261320:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572470.783994:0:261320:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572470.783995:0:261320:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572470.783997:0:261320:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572470.784000:0:261320:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572470.784004:0:261320:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.784007:0:261320:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572470.784012:0:261320:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572470.784015:0:261320:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572470.784019:0:261320:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.784022:0:261320:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.784026:0:261320:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.784028:0:261320:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.784029:0:261320:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.784032:0:261320:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.784034:0:261320:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.784037:0:261320:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.784042:0:261320:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6600. 00000100:00000001:0.0:1788572470.784049:0:261320:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.784053:0:261320:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.784055:0:261320:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.784056:0:261320:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.784057:0:261320:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.784059:0:261320:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.784061:0:261320:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00010000:00000001:0.0:1788572470.784064:0:261320:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.784065:0:261320:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572470.784069:0:261320:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76200. 00010000:00000010:0.0:1788572470.784072:0:261320:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3708c76b00. 00010000:00000001:0.0:1788572470.784076:0:261320:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.784079:0:261320:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6a600. 00000020:00000001:0.0:1788572470.784083:0:261320:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.784085:0:261320:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6a600 with handle 0x144158609316843e to hash 00000020:00000001:0.0:1788572470.784087:0:261320:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.784088:0:261320:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553697280 : -109714155854336 : ffff9c372eb6a600) 00010000:00000010:0.0:1788572470.784092:0:261320:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091f00. 00010000:00000001:0.0:1788572470.784094:0:261320:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553697280 : -109714155854336 : ffff9c372eb6a600) 00010000:00000001:0.0:1788572470.784097:0:261320:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.784099:0:261320:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.784102:0:261320:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000f9053d9f/0x144158609316843e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5267:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261320 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.784112:0:261320:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000f9053d9f/0x144158609316843e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5267:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261320 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.784128:0:261320:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.784132:0:261320:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.784136:0:261320:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.784144:0:261320:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b7480 x1875440809937920/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572470.784168:0:261320:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572470.784171:0:261320:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.784173:0:261320:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.784175:0:261320:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.784176:0:261320:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.784179:0:261320:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572470.784180:0:261320:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572470.784182:0:261320:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572470.784184:0:261320:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000800:00000200:0.0:1788572470.786091:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.786096:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572470.786101:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.786106:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.786112:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788572470.786114:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788572470.786118:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.786122:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7480 x1875440809937920/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.786132:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.786135:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440809937920/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.786141:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.786143:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.789332:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.789349:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.789352:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.789356:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.789364:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.789372:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.789381:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.789385:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.789390:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.789394:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460c00 00000400:00000200:0.0:1788572470.789399:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.789404:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529b11 [1] + 224 00000800:00000001:0.0:1788572470.789410:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.789426:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.789430:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.789435:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.789443:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.789452:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7480 x1875440809937920/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.789463:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c372d6b7480 x1875440809937920/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.789472:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.789480:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.789484:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.792269:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.792279:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572470.793122:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.793133:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.793141:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572470.793144:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572470.793150:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.793158:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.793169:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.793172:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.793180:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.793181:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.795184:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.795198:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.795200:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.795203:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.795210:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.795216:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.795225:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.795229:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.795235:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.795239:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460c80 00000400:00000200:0.0:1788572470.795243:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.795249:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529b21 [1] + 224 00000800:00000001:0.0:1788572470.795255:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.795261:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.795264:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.795268:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.795275:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.795281:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.795292:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.795308:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.795315:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.795317:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.795341:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.795345:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.795352:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.795353:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.795356:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f29f8. 00000400:00000200:0.0:1788572470.795360:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3330 00000400:00000010:0.0:1788572470.795364:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3330. 00000100:00000001:0.0:1788572470.795367:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.795371:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572470.795379:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.795385:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.795387:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.795389:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.795392:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.795395:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.795397:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.795400:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.795409:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce538 time=23 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.795428:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.795432:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.795434:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.795437:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.795440:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.795444:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.795447:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.795449:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.795452:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.795457:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.795465:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.795467:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.795469:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.795473:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572470.795481:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809938048:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.795486:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.795497:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.795500:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440809938048/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.795506:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.795509:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.795513:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3711215c00. 02000000:00000001:0.0:1788572470.795518:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.795521:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.795523:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.795525:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.795527:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215e00. 02000000:00000001:0.0:1788572470.795530:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.795532:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.795534:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788572470.795537:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.795538:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.795541:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.795549:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.795551:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.795553:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.795555:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.822815:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.822825:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.822828:0:261322:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.822832:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fb20 00000080:00200000:0.0:1788572470.822836:0:261322:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.822846:0:261322:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572470.822855:0:261322:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.822859:0:261322:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.822865:0:261322:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.822868:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fa40 00800000:00000002:0.0:1788572470.822870:0:261322:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.822874:0:261322:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.822887:0:261322:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.822890:0:261322:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.822893:0:261322:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.822895:0:261322:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.822899:0:261322:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.822901:0:261322:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.822903:0:261322:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.822907:0:261322:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.822912:0:261322:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.822914:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f910 00000002:00010000:0.0:1788572470.822916:0:261322:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.822920:0:261322:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.822922:0:261322:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.822925:0:261322:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.822932:0:261322:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.822936:0:261322:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.822941:0:261322:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.822945:0:261322:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.822950:0:261322:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 148 lru_type: 1 02000000:00000001:0.0:1788572470.822962:0:261322:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.822967:0:261322:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.822970:0:261322:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 148 lru_type: 1 00000002:00000001:0.0:1788572470.822979:0:261322:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.822982:0:261322:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.822983:0:261322:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.822985:0:261322:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.822987:0:261322:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.822989:0:261322:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.822992:0:261322:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.822995:0:261322:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.822998:0:261322:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6000. 00000080:00000001:0.0:1788572470.823008:0:261322:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.823009:0:261322:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.823012:0:261322:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.823016:0:261322:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.823019:0:261322:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.823022:0:261322:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.823024:0:261322:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.823026:0:261322:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.823030:0:261322:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.823032:0:261322:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.823035:0:261322:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.823038:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.823039:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.823040:0:261322:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.823043:0:261322:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.823045:0:261322:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c9fb58 00010000:00000001:0.0:1788572470.823047:0:261322:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.823048:0:261322:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.823050:0:261322:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.823052:0:261322:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.823054:0:261322:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.823056:0:261322:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.823059:0:261322:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 148 lru_type: 1 00010000:00000001:0.0:1788572470.823067:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.823068:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.823751:0:261322:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.823756:0:261322:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.823760:0:261322:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 149 lru_type: 1 00010000:00000001:0.0:1788572470.823769:0:261322:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.823772:0:261322:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.823774:0:261322:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.823779:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.823781:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.823783:0:261322:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823786:0:261322:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.823788:0:261322:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823790:0:261322:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823793:0:261322:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823795:0:261322:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.823796:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.823797:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.823800:0:261322:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.823802:0:261322:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.823804:0:261322:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.823810:0:261322:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.823816:0:261322:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.823819:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fae8 00000080:00002000:0.0:1788572470.823821:0:261322:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 24 00000080:00000001:0.0:1788572470.823823:0:261322:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.823825:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fb30 00000080:00200000:0.0:1788572470.823826:0:261322:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.823832:0:261322:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.823837:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.823839:0:261322:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.823842:0:261322:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.823854:0:261322:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f340. 00000020:00000001:0.0:1788572470.823858:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fae8 00000080:00200000:0.0:1788572470.823860:0:261322:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-025, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.823863:0:261322:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.823865:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f998 00000080:00200000:0.0:1788572470.823867:0:261322:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-025, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.823873:0:261322:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572470.823889:0:261322:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.823895:0:261322:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.823898:0:261322:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.823900:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f8b8 00800000:00000002:0.0:1788572470.823901:0:261322:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-025' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.823904:0:261322:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.823907:0:261322:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.823909:0:261322:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.823910:0:261322:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.823912:0:261322:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.823914:0:261322:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.823915:0:261322:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.823917:0:261322:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.823920:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f850 00800000:00000002:0.0:1788572470.823922:0:261322:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-025' -> mds #0 00000002:00000001:0.0:1788572470.823925:0:261322:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.823927:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f788 00000002:00010000:0.0:1788572470.823929:0:261322:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-025,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.823933:0:261322:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.823935:0:261322:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.823940:0:261322:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572470.823945:0:261322:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.823950:0:261322:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.823954:0:261322:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.823957:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.823958:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.823959:0:261322:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.823962:0:261322:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.823963:0:261322:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.823967:0:261322:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.823974:0:261322:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572470.823984:0:261322:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823989:0:261322:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.823992:0:261322:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823993:0:261322:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.823994:0:261322:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.823996:0:261322:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.824004:0:261322:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00010000:00000001:0.0:1788572470.824008:0:261322:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.824010:0:261322:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.824018:0:261322:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.824020:0:261322:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.824024:0:261322:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.824028:0:261322:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b000. 00000020:00000001:0.0:1788572470.824033:0:261322:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.824036:0:261322:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b000 with handle 0x1441586093168445 to hash 00000020:00000001:0.0:1788572470.824039:0:261322:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.824040:0:261322:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.824043:0:261322:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572470.824047:0:261322:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.824049:0:261322:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.824052:0:261322:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168445 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.824059:0:261322:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168445 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.824407:0:261322:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b000/0x1441586093168445 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261322 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.824417:0:261322:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.824419:0:261322:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.824427:0:261322:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5cc0. 00000100:00000001:0.0:1788572470.824429:0:261322:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716480 : -109714420835136 : ffff9c371eeb5cc0) 00000100:00000001:0.0:1788572470.824431:0:261322:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.824433:0:261322:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788572470.824438:0:261322:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.824442:0:261322:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.824444:0:261322:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.824446:0:261322:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.824457:0:261322:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809938176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.824465:0:261322:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.824466:0:261322:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.824470:0:261322:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.824473:0:261322:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.824476:0:261322:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.824480:0:261322:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261322:1875440809938176:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.824484:0:261322:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.824489:0:261322:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.824492:0:261322:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.824494:0:261322:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.824499:0:261322:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e8000. 02000000:00000001:0.0:1788572470.824500:0:261322:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.824510:0:261322:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.824514:0:261322:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8630. 00000400:00000010:0.0:1788572470.824522:0:261322:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788572470.824529:0:261322:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809938176, portal 10 00000100:00000001:0.0:1788572470.824531:0:261322:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.824532:0:261322:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572470.824538:0:261322:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4700 x1875440809938176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.824546:0:261322:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.824548:0:261322:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.824551:0:261322:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572470.824554:0:261322:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809938176, offset 0 00000400:00000200:0.0:1788572470.824560:0:261322:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.824570:0:261322:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.824576:0:261322:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.824582:0:261322:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.824586:0:261322:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.824589:0:261322:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.824591:0:261322:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.824595:0:261322:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539709:539709:256:4294967295] 192.168.202.148@tcp LPNI seq info [539674:539674:8:4294967295] 00000400:00000200:0.0:1788572470.824599:0:261322:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.824608:0:261322:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.824614:0:261322:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.824620:0:261322:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572470.824626:0:261322:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.824665:0:261322:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.824671:0:261322:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.824689:0:261322:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.824695:0:261322:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14988 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.824698:0:261322:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.824701:0:261322:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609845 00000100:00000001:0.0:1788572470.824705:0:261322:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307609964 00000100:00000001:0.0:1788572470.824708:0:261322:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609845 00000100:00000001:0.0:1788572470.824710:0:261322:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307609964 00000100:00000001:0.0:1788572470.824712:0:261322:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.824715:0:261322:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.824718:0:261322:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.824721:0:261322:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.824723:0:261322:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5cc0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.824726:0:261322:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.824729:0:261322:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.828677:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.828697:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.828701:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.828704:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.828714:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.828723:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.828733:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.828739:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.828745:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.828749:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417460d00 00000400:00000200:0.0:1788572470.828755:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.828763:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529b31 [1] + 224 00000800:00000001:0.0:1788572470.828770:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.828779:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.828783:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.828788:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.828796:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.828806:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4700 x1875440809938176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.828817:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4700 x1875440809938176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.828840:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.828849:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.828853:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.831549:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.831553:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.839959:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.842554:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.842564:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.843208:0:261322:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843211:0:261322:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.843217:0:261322:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572470.843220:0:261322:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e6c0 -> ffff9c371ee2e6c0 00000020:00000001:0.0:1788572470.843222:0:261322:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e6c0 -> ffff9c371ee2e6c0 00000020:00000001:0.0:1788572470.843223:0:261322:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572470.843228:0:261322:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572470.843232:0:261322:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.843234:0:261322:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572470.843236:0:261322:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572470.843237:0:261322:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.843239:0:261322:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdad:11031 on OST[1]: s=0 (1) m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.843243:0:261322:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572470.843245:0:261322:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572470.843247:0:261322:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843249:0:261322:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843250:0:261322:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e6c0 -> ffff9c371ee2e6c0 00000080:00200000:0.0:1788572470.843253:0:261322:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdad:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572470.843255:0:261322:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572470.843258:0:261322:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572470.843260:0:261322:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572470.843263:0:261322:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.843264:0:261322:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572470.843267:0:261322:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572470.843271:0:261322:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572470.843272:0:261322:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572470.843274:0:261322:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572470.843275:0:261322:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192a2b0 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572470.843278:0:261322:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572470.843280:0:261322:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572470.843281:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.843282:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572470.843284:0:261322:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572470.843286:0:261322:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.843287:0:261322:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.843288:0:261322:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572470.843289:0:261322:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.843290:0:261322:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572470.843293:0:261322:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572470.843294:0:261322:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572470.843294:0:261322:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192a2b0 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572470.843297:0:261322:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313a000. 00000020:00000001:0.0:1788572470.843300:0:261322:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572470.843302:0:261322:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192a280. 00020000:00000001:0.0:1788572470.843316:0:261322:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.843317:0:261322:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572470.843318:0:261322:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572470.843320:0:261322:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843323:0:261322:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572470.843325:0:261322:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572470.843327:0:261322:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572470.843328:0:261322:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572470.843330:0:261322:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.843332:0:261322:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572470.843335:0:261322:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572470.843337:0:261322:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572470.843339:0:261322:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572470.843342:0:261322:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdad:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572470.843346:0:261322:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572470.843348:0:261322:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.843350:0:261322:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572470.843352:0:261322:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572470.843352:0:261322:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572470.843354:0:261322:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572470.843356:0:261322:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843363:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fd08 00000080:00000008:0.0:1788572470.843366:0:261322:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-025:[0x200002b17:0xdad:0x0](ffff9c3719f83248), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572470.843372:0:261322:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.843373:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fda0 00000080:00002000:0.0:1788572470.843375:0:261322:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-025 (ffff9c3719f64bd0, parent ffff9c3719d93af8, inode ffff9c3719f83248) hashed, 00000080:00000001:0.0:1788572470.843378:0:261322:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.843407:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.843412:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.843414:0:261322:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.843416:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fbc0 00000080:00200000:0.0:1788572470.843418:0:261322:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.843423:0:261322:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572470.843430:0:261322:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.843434:0:261322:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.843437:0:261322:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.843439:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fae0 00800000:00000002:0.0:1788572470.843441:0:261322:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.843445:0:261322:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.843447:0:261322:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.843450:0:261322:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.843453:0:261322:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.843456:0:261322:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.843458:0:261322:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.843460:0:261322:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.843462:0:261322:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.843465:0:261322:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.843470:0:261322:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.843472:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9f9b0 00000002:00010000:0.0:1788572470.843473:0:261322:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.843477:0:261322:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.843479:0:261322:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.843481:0:261322:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.843489:0:261322:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.843492:0:261322:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.843496:0:261322:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.843500:0:261322:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.843505:0:261322:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 149 lru_type: 1 02000000:00000001:0.0:1788572470.843513:0:261322:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.843518:0:261322:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.843521:0:261322:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 149 lru_type: 1 00000002:00000001:0.0:1788572470.843527:0:261322:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.843529:0:261322:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.843531:0:261322:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.843533:0:261322:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.843534:0:261322:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.843536:0:261322:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.843539:0:261322:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.843541:0:261322:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.843544:0:261322:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572470.843550:0:261322:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.843551:0:261322:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.843554:0:261322:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.843557:0:261322:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.843561:0:261322:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.843563:0:261322:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.843564:0:261322:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.843566:0:261322:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.843568:0:261322:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.843570:0:261322:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.843572:0:261322:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.843575:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.843576:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.843577:0:261322:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.843579:0:261322:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.843581:0:261322:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c9fbf8 00010000:00000001:0.0:1788572470.843583:0:261322:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.843584:0:261322:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.843585:0:261322:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.843586:0:261322:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.843588:0:261322:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.843590:0:261322:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.843592:0:261322:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 149 lru_type: 1 00010000:00000001:0.0:1788572470.843598:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.843599:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.843600:0:261322:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.843601:0:261322:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.843603:0:261322:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 150 lru_type: 1 00010000:00000001:0.0:1788572470.843609:0:261322:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.843611:0:261322:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.843612:0:261322:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.843615:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.843615:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.843617:0:261322:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.843619:0:261322:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.843620:0:261322:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.843622:0:261322:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.843625:0:261322:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.843626:0:261322:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.843627:0:261322:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.843629:0:261322:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.843630:0:261322:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.843632:0:261322:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.843666:0:261322:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.843670:0:261322:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.843674:0:261322:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.843676:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fb88 00000080:00002000:0.0:1788572470.843677:0:261322:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 25 00000080:00000001:0.0:1788572470.843680:0:261322:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843682:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fbd0 00000080:00200000:0.0:1788572470.843683:0:261322:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572470.843688:0:261322:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.843692:0:261322:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.843693:0:261322:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.843696:0:261322:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.843699:0:261322:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.843700:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fbe8 00000080:00002000:0.0:1788572470.843701:0:261322:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-025(ffff9c3719f64bd0) flags 0x48000e refc 0 00000080:00000001:0.0:1788572470.843703:0:261322:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.843704:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fc30 00000080:00200000:0.0:1788572470.843705:0:261322:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-025, flags=9 00000080:00000001:0.0:1788572470.843717:0:261322:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572470.843719:0:261322:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572470.843722:0:261322:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572470.843723:0:261322:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c9fe40 00000080:00002000:0.0:1788572470.843725:0:261322:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-025 (ffff9c3719f64bd0, parent ffff9c3719d93af8, inode ffff9c3719f83248) hashed, 00000080:00000001:0.0:1788572470.843727:0:261322:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.848266:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.848280:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.848283:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.848286:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.848294:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.848302:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.848311:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.848316:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.848321:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.848326:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460e00 00000400:00000200:0.0:1788572470.848331:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.848338:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529b51 [1] + 224 00000800:00000001:0.0:1788572470.848346:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.848353:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.848357:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.848361:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.848369:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.848378:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809938432/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.848390:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69880 x1875440809938432/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.848411:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.848420:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.848424:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572470.859773:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.859778:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572470.859783:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.859788:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.859794:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c14c8 00000400:00000010:0.0:1788572470.859796:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c14c8. 00000100:00000001:0.0:1788572470.859800:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.859804:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809938560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.859813:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.859816:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72680 x1875440809938560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.859822:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.859824:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.868006:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.868025:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.868030:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.868032:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.868043:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.868051:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.868059:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.868063:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.868068:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.868072:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417460e80 00000400:00000200:0.0:1788572470.868078:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.868084:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529b59 [1] + 224 00000800:00000001:0.0:1788572470.868091:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.868098:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.868102:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.868106:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.868114:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.868124:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809938560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.868134:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec72680 x1875440809938560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.868161:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.868169:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.868173:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.870626:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.870629:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870632:0:248770:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.870647:0:248770:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.870649:0:248770:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.870653:0:248770:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.870661:0:248770:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c371f010e00 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.870663:0:248770:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.870667:0:248770:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.870677:0:248770:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.870679:0:248770:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.870682:0:248770:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.870688:0:248770:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.870690:0:248770:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.870691:0:248770:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.870693:0:248770:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.870695:0:248770:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f010e00 refcount=3 00000020:00000001:0.0:1788572470.870697:0:248770:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359290138112 : -109714419413504 : ffff9c371f010e00) 00010000:00000001:0.0:1788572470.870700:0:248770:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359290138112 : -109714419413504 : ffff9c371f010e00) 00010000:00000001:0.0:1788572470.870702:0:248770:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.870704:0:248770:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.870710:0:248770:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.870713:0:248770:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.870715:0:248770:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.870717:0:248770:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xdae:0x0] mode (null) 00000002:00000001:0.0:1788572470.870720:0:248770:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.870721:0:248770:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.870726:0:248770:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7400 count: 4 00010000:00000001:0.0:1788572470.870729:0:248770:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.870731:0:248770:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.870733:0:248770:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.870735:0:248770:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.870737:0:248770:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870740:0:248770:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119400/0x1441586093168381 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a200d expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00000001:0.0:1788572470.870751:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.870752:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.870753:0:248770:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.870754:0:248770:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.870756:0:248770:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.870757:0:248770:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c371f119400 refcount=2 00000020:00000001:0.0:1788572470.870758:0:248770:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359291220992 : -109714418330624 : ffff9c371f119400) 00010000:00000001:0.0:1788572470.870760:0:248770:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359291220992 : -109714418330624 : ffff9c371f119400) 00010000:00000001:0.0:1788572470.870762:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.870763:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.870764:0:248770:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.870766:0:248770:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.870767:0:248770:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.870770:0:248770:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.870772:0:248770:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.870774:0:248770:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.870776:0:248770:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.870777:0:248770:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.870779:0:248770:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f010e00 with handle 0x1441586093168453 from hash 00010000:00000001:0.0:1788572470.870781:0:248770:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.870782:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.870784:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.870784:0:248770:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.870787:0:248770:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.870789:0:248770:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.870793:0:248770:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.870795:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.870796:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.870797:0:248770:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.870799:0:248770:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.870801:0:248770:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.870806:0:248770:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572470.870809:0:248770:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.870812:0:248770:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.870815:0:248770:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.870817:0:248770:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.870822:0:248770:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6e00. 00000100:00000001:0.0:1788572470.870826:0:248770:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.870828:0:248770:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.870832:0:248770:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.870838:0:248770:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.870840:0:248770:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.870842:0:248770:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.870845:0:248770:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.870851:0:248770:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.870872:0:248770:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.870874:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.870889:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.870891:0:248770:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.870893:0:248770:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.870895:0:248770:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870898:0:248770:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.870903:0:248770:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.870906:0:248770:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f010e00/0x1441586093168453 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xdae:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a21f7 expref: -99 pid: 261324 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.870912:0:248770:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372eccb3c0. 00010000:00000001:0.0:1788572470.870920:0:248770:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.870922:0:248770:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.870924:0:248770:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c37380b6f00. 00010000:00000001:0.0:1788572470.870927:0:248770:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.870929:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.870930:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870932:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.870934:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.870935:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870937:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.870938:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.870939:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.870941:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572470.871037:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.871041:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572470.871046:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.871051:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.871056:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf15d8 00000400:00000010:0.0:1788572470.871058:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf15d8. 00000100:00000001:0.0:1788572470.871062:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.871066:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809938688/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.871074:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.871077:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809938688/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.871082:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.871084:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.871097:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572470.871100:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572470.871102:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.871106:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.871108:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.871110:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.871111:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.871112:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.871117:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.871123:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.871124:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.871127:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.871129:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.871131:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.871133:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809938816:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.871137:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.871140:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.871142:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.871143:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.871147:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572470.871149:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.871153:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.871157:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b00. 00000400:00000010:0.0:1788572470.871161:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572470.871165:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809938816, portal 18 00000100:00000001:0.0:1788572470.871166:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.871168:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572470.871173:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.871179:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.871181:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.871183:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572470.871185:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809938816, offset 0 00000400:00000200:0.0:1788572470.871190:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.871194:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.871199:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.871202:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.871204:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.871206:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.871209:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.871211:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539714:539714:256:4294967295] 192.168.202.148@tcp LPNI seq info [539679:539679:8:4294967295] 00000400:00000200:0.0:1788572470.871215:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.871220:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.871225:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.871229:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572470.871233:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.871238:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.871240:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.871248:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.871252:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14988 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.871255:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.871257:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609889 00000100:00000001:0.0:1788572470.871259:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.871262:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609889 00000100:00000001:0.0:1788572470.871264:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.871266:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.871268:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.871270:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.871272:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.871290:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.871293:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572470.871296:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.871300:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.871303:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31f68 00000400:00000010:0.0:1788572470.871305:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31f68. 00000100:00000001:0.0:1788572470.871307:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.871311:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.871317:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.871320:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.871325:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.871326:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.871334:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.871336:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.876435:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.876456:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.876460:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.876463:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.876472:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.876480:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.876490:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.876495:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.876501:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.876505:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417460f00 00000400:00000200:0.0:1788572470.876510:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.876516:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529b61 [1] + 224 00000800:00000001:0.0:1788572470.876523:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.876529:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.876533:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.876538:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.876545:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.876555:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809938688/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.876566:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69880 x1875440809938688/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.876575:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.876582:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.876585:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.880790:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.880810:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.880815:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.880818:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.880829:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.880838:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.880850:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.880856:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.880862:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.880868:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417460f80 00000400:00000200:0.0:1788572470.880874:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.880891:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x529b69 [1] + 224 00000800:00000001:0.0:1788572470.880899:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.880955:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.880959:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.880964:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.880971:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.880980:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572470.880992:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.881010:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.881021:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.881025:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.881038:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.881043:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.881050:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.881051:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.881054:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8b00. 00000400:00000200:0.0:1788572470.881059:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788572470.881061:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788572470.881064:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.881068:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572470.881076:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.881082:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.881085:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.881087:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.881091:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.881093:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.881096:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.881099:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.881109:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce538 time=23 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.881113:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.881116:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.881117:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.881120:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.881123:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.881127:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.881135:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.881137:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.881141:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.881146:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.881153:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.881156:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.881158:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.881162:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572470.881170:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809938816:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.881175:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.881179:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.881182:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4000 x1875440809938816/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.881188:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.881191:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.881194:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572470.881201:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.881204:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.881206:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.881209:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.881211:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b6e00. 02000000:00000001:0.0:1788572470.881217:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.881220:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.881222:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4000. 00000100:00000001:0.0:1788572470.881225:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.881226:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.881228:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.881236:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.881239:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.881241:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.881243:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.898555:0:261326:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.898563:0:261326:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.898566:0:261326:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.898569:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37b20 00000080:00200000:0.0:1788572470.898574:0:261326:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.898582:0:261326:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.898589:0:261326:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.898593:0:261326:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.898599:0:261326:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.898601:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37a40 00800000:00000002:0.0:1788572470.898603:0:261326:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.898607:0:261326:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.898610:0:261326:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.898612:0:261326:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.898614:0:261326:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.898617:0:261326:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.898619:0:261326:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.898621:0:261326:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.898623:0:261326:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.898626:0:261326:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.898632:0:261326:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.898846:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37910 00000002:00010000:0.0:1788572470.898850:0:261326:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.898853:0:261326:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.898855:0:261326:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.898859:0:261326:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.898867:0:261326:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.898870:0:261326:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.898886:0:261326:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.898891:0:261326:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.898896:0:261326:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 152 lru_type: 1 02000000:00000001:0.0:1788572470.898908:0:261326:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.898913:0:261326:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.898917:0:261326:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 152 lru_type: 1 00000002:00000001:0.0:1788572470.898924:0:261326:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.898926:0:261326:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.898928:0:261326:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.898930:0:261326:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.898933:0:261326:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.898935:0:261326:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.898938:0:261326:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.898941:0:261326:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.898945:0:261326:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572470.898953:0:261326:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.898955:0:261326:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.898958:0:261326:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.898961:0:261326:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.898965:0:261326:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.898967:0:261326:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.898969:0:261326:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.898971:0:261326:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.898974:0:261326:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.898977:0:261326:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.898980:0:261326:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.898983:0:261326:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.898984:0:261326:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.898985:0:261326:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.898987:0:261326:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.898990:0:261326:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1c37b58 00010000:00000001:0.0:1788572470.898992:0:261326:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.898993:0:261326:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.898994:0:261326:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.898996:0:261326:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.898998:0:261326:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.899000:0:261326:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.899003:0:261326:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 152 lru_type: 1 00010000:00000001:0.0:1788572470.899010:0:261326:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.899011:0:261326:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.899013:0:261326:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.899015:0:261326:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.899017:0:261326:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 153 lru_type: 1 00010000:00000001:0.0:1788572470.899024:0:261326:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.899028:0:261326:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.899030:0:261326:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.899034:0:261326:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.899035:0:261326:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.899037:0:261326:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899041:0:261326:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.899043:0:261326:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899045:0:261326:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899048:0:261326:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899051:0:261326:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.899052:0:261326:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.899053:0:261326:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.899057:0:261326:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.899058:0:261326:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.899061:0:261326:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.899065:0:261326:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.899069:0:261326:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.899072:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37ae8 00000080:00002000:0.0:1788572470.899073:0:261326:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 26 00000080:00000001:0.0:1788572470.899076:0:261326:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.899078:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37b30 00000080:00200000:0.0:1788572470.899079:0:261326:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.899084:0:261326:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.899088:0:261326:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.899089:0:261326:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.899092:0:261326:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.899101:0:261326:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310ff60. 00000020:00000001:0.0:1788572470.899106:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37ae8 00000080:00200000:0.0:1788572470.899108:0:261326:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-027, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.899111:0:261326:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.899113:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37998 00000080:00200000:0.0:1788572470.899115:0:261326:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-027, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.899121:0:261326:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572470.899124:0:261326:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.899129:0:261326:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.899130:0:261326:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.899132:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c378b8 00800000:00000002:0.0:1788572470.899134:0:261326:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-027' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.899136:0:261326:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.899139:0:261326:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.899141:0:261326:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.899142:0:261326:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.899143:0:261326:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.899145:0:261326:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.899146:0:261326:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.899149:0:261326:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.899151:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37850 00800000:00000002:0.0:1788572470.899152:0:261326:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-027' -> mds #0 00000002:00000001:0.0:1788572470.899156:0:261326:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.899157:0:261326:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c37788 00000002:00010000:0.0:1788572470.899158:0:261326:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-027,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.899162:0:261326:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.899165:0:261326:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.899170:0:261326:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572470.899174:0:261326:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.899179:0:261326:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.899183:0:261326:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.899186:0:261326:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.899188:0:261326:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.899189:0:261326:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.899192:0:261326:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.899195:0:261326:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.899198:0:261326:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.899205:0:261326:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572470.899212:0:261326:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899218:0:261326:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.899220:0:261326:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899221:0:261326:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.899223:0:261326:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.899224:0:261326:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.899232:0:261326:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00010000:00000001:0.0:1788572470.899236:0:261326:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.899238:0:261326:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.899241:0:261326:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.899243:0:261326:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.899246:0:261326:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.899249:0:261326:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b400. 00000020:00000001:0.0:1788572470.899253:0:261326:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.899255:0:261326:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b400 with handle 0x1441586093168461 to hash 00000020:00000001:0.0:1788572470.899257:0:261326:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.899259:0:261326:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.899263:0:261326:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.899265:0:261326:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.899267:0:261326:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.899271:0:261326:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261326 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.899283:0:261326:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261326 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.899294:0:261326:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261326 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.899299:0:261326:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.899301:0:261326:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.899308:0:261326:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5cc0. 00000100:00000001:0.0:1788572470.899311:0:261326:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716480 : -109714420835136 : ffff9c371eeb5cc0) 00000100:00000001:0.0:1788572470.899314:0:261326:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.899315:0:261326:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000001:0.0:1788572470.899320:0:261326:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.899324:0:261326:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.899326:0:261326:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.899328:0:261326:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.899340:0:261326:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.899348:0:261326:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.899350:0:261326:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.899353:0:261326:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.899356:0:261326:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.899359:0:261326:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.899362:0:261326:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261326:1875440809938944:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.899366:0:261326:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.899371:0:261326:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.899373:0:261326:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.899376:0:261326:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.899381:0:261326:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ed800. 02000000:00000001:0.0:1788572470.899383:0:261326:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.899390:0:261326:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.899395:0:261326:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b00. 00000400:00000010:0.0:1788572470.899402:0:261326:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572470.899409:0:261326:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809938944, portal 10 00000100:00000001:0.0:1788572470.899411:0:261326:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.899412:0:261326:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572470.899419:0:261326:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.899426:0:261326:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.899429:0:261326:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.899432:0:261326:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572470.899435:0:261326:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809938944, offset 0 00000400:00000200:0.0:1788572470.899441:0:261326:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.899450:0:261326:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.899458:0:261326:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.899463:0:261326:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.899467:0:261326:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.899471:0:261326:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.899474:0:261326:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.899478:0:261326:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539715:539715:256:4294967295] 192.168.202.148@tcp LPNI seq info [539680:539680:8:4294967295] 00000400:00000200:0.0:1788572470.899482:0:261326:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.899491:0:261326:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.899497:0:261326:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.899503:0:261326:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572470.899509:0:261326:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.899516:0:261326:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.899520:0:261326:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.899588:0:261326:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.899593:0:261326:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14987 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.899598:0:261326:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.899601:0:261326:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609920 00000100:00000001:0.0:1788572470.899604:0:261326:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.899606:0:261326:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609920 00000100:00000001:0.0:1788572470.899609:0:261326:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.899611:0:261326:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.899614:0:261326:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.899616:0:261326:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.899618:0:261326:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.899620:0:261326:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5cc0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.899624:0:261326:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.899626:0:261326:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.902212:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.902222:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572470.902228:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.902237:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.902244:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31f68 00000400:00000010:0.0:1788572470.902247:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31f68. 00000100:00000001:0.0:1788572470.902252:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.902260:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.902272:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.902275:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.902282:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.902284:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572470.909929:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.909947:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.909952:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.909955:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.909963:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.909971:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.909981:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.909985:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.909991:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.909995:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461000 00000400:00000200:0.0:1788572470.910001:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.910008:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529b79 [1] + 224 00000800:00000001:0.0:1788572470.910015:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.910023:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.910026:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.910030:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.910038:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.910047:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.910057:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4000 x1875440809938944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.910088:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.910096:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.910100:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.911180:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.911183:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911186:0:237083:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572470.911190:0:237083:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572470.911192:0:237083:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572470.911195:0:237083:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.911204:0:237083:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372eb6b400 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572470.911207:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572470.911211:0:237083:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.911218:0:237083:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572470.911220:0:237083:0:(ldlm_inodebits.c:543:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572470.911223:0:237083:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.911233:0:237083:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572470.911235:0:237083:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572470.911236:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.911238:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.911239:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b400 refcount=3 00000020:00000001:0.0:1788572470.911241:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.911243:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.911245:0:237083:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572470.911249:0:237083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 3/0,0 mode: PR/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572470.911255:0:237083:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572470.911259:0:237083:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572470.911260:0:237083:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572470.911262:0:237083:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xdaf:0x0] mode (null) 00000002:00000001:0.0:1788572470.911265:0:237083:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.911266:0:237083:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.911270:0:237083:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25e00 count: 4 00010000:00000001:0.0:1788572470.911272:0:237083:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.911275:0:237083:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered 00010000:00000001:0.0:1788572470.911277:0:237083:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving 02000000:00000001:0.0:1788572470.911279:0:237083:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.911281:0:237083:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911284:0:237083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c373204b200/0x144158609316838f lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1f49 expref: -99 pid: 261294 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572470.911291:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.911292:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.911294:0:237083:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00010000:00000001:0.0:1788572470.911296:0:237083:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.911297:0:237083:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.911298:0:237083:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204b200 refcount=4 00000020:00000001:0.0:1788572470.911300:0:237083:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609143808 : -109714100407808 : ffff9c373204b200) 00010000:00000001:0.0:1788572470.911302:0:237083:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609143808 : -109714100407808 : ffff9c373204b200) 00010000:00000001:0.0:1788572470.911304:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.911305:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572470.911306:0:237083:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572470.911309:0:237083:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572470.911311:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.911313:0:237083:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572470.911315:0:237083:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572470.911316:0:237083:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572470.911318:0:237083:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.911320:0:237083:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572470.911322:0:237083:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372eb6b400 with handle 0x1441586093168461 from hash 00010000:00000001:0.0:1788572470.911324:0:237083:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572470.911325:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.911326:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.911327:0:237083:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572470.911330:0:237083:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572470.911332:0:237083:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572470.911336:0:237083:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.911340:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.911341:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.911342:0:237083:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.911344:0:237083:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.911346:0:237083:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572470.911353:0:237083:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572470.911356:0:237083:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=44 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572470.911360:0:237083:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.911362:0:237083:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.911365:0:237083:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.911371:0:237083:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7200. 00000100:00000001:0.0:1788572470.911377:0:237083:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.911380:0:237083:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572470.911384:0:237083:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 2/0,0 mode: --/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.911397:0:237083:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572470.911398:0:237083:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.911403:0:237083:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.911407:0:237083:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.911416:0:237083:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572470.911450:0:237083:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.911452:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.911454:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.911455:0:237083:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.911457:0:237083:0:(ldlm_request.c:1834:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.911459:0:237083:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911462:0:237083:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572470.911468:0:237083:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572470.911471:0:237083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x1441586093168461 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xdaf:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a221a expref: -99 pid: 261326 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000010:0.0:1788572470.911478:0:237083:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372eccbc00. 00010000:00000001:0.0:1788572470.911485:0:237083:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.911487:0:237083:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1788572470.911489:0:237083:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c37380b6600. 00010000:00000001:0.0:1788572470.911492:0:237083:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.911494:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.911495:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911497:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.911499:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.911500:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911501:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572470.911503:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572470.911505:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.911506:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572470.911515:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572470.911518:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572470.911520:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.911528:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.911530:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.911532:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.911533:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.911535:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.911540:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.911548:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.911561:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.911565:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.911567:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.911570:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.911573:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809939072:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.911577:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.911582:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.911584:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.911586:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.911590:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c37101b6800. 02000000:00000001:0.0:1788572470.911592:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.911599:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.911603:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572470.911609:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572470.911616:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809939072, portal 18 00000100:00000001:0.0:1788572470.911618:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.911620:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000040:0.0:1788572470.911625:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.911631:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.911634:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.911678:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572470.911680:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1875440809939072, offset 0 00000400:00000200:0.0:1788572470.911686:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.911693:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.911699:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.911704:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.911708:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.911712:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.911715:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.911718:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539716:539716:256:4294967295] 192.168.202.148@tcp LPNI seq info [539681:539681:8:4294967295] 00000400:00000200:0.0:1788572470.911722:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.911729:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.911734:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.911739:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572470.911744:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.911751:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.911755:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.911765:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.911770:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14987 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.911773:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.911776:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609932 00000100:00000001:0.0:1788572470.911779:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.911781:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609932 00000100:00000001:0.0:1788572470.911783:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610046 00000100:00000001:0.0:1788572470.911785:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.911788:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.911791:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.911793:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572470.911902:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572470.911906:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572470.911911:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.911915:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572470.911920:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788572470.911922:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788572470.911925:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572470.911929:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.911939:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.911943:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.911949:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.911951:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572470.915996:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.915998:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572470.916004:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572470.916005:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572470.916008:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c81b8. 00000400:00000200:0.0:1788572470.916011:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31f68 00000400:00000010:0.0:1788572470.916013:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31f68. 00000100:00000001:0.0:1788572470.916015:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.916020:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572470.916028:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.916034:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572470.916035:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.916038:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572470.916040:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572470.916042:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572470.916044:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.916046:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572470.916054:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce538 time=23 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572470.916058:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=66 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572470.916060:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572470.916064:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.916066:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572470.916070:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.916073:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572470.916075:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572470.916078:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572470.916081:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.916085:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.916092:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.916095:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572470.916097:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572470.916101:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572470.916108:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809939072:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572470.916112:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.916114:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572470.916117:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6300 x1875440809939072/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572486 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572470.916124:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572470.916125:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572470.916128:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37101b6800. 02000000:00000001:0.0:1788572470.916134:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572470.916136:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572470.916138:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572470.916139:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572470.916142:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572470.916147:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572470.916148:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572470.916150:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6300. 00000100:00000001:0.0:1788572470.916153:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572470.916154:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.916156:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.916161:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.916162:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.916164:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.916166:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.949250:0:261328:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.949259:0:261328:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572470.949262:0:261328:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572470.949265:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fb20 00000080:00200000:0.0:1788572470.949270:0:261328:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572470.949279:0:261328:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572470.949286:0:261328:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572470.949290:0:261328:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.949295:0:261328:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.949298:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fa40 00800000:00000002:0.0:1788572470.949301:0:261328:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.949305:0:261328:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.949307:0:261328:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.949310:0:261328:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.949313:0:261328:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.949316:0:261328:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.949319:0:261328:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.949321:0:261328:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572470.949324:0:261328:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.949328:0:261328:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572470.949333:0:261328:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.949336:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f910 00000002:00010000:0.0:1788572470.949337:0:261328:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572470.949341:0:261328:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572470.949343:0:261328:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572470.949347:0:261328:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572470.949354:0:261328:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572470.949357:0:261328:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.949362:0:261328:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.949366:0:261328:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572470.949372:0:261328:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 154 lru_type: 1 02000000:00000001:0.0:1788572470.949388:0:261328:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572470.949394:0:261328:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.949397:0:261328:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 154 lru_type: 1 00000002:00000001:0.0:1788572470.949403:0:261328:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572470.949406:0:261328:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572470.949408:0:261328:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.949410:0:261328:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572470.949412:0:261328:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.949414:0:261328:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572470.949417:0:261328:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572470.949420:0:261328:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572470.949423:0:261328:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572470.949431:0:261328:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572470.949433:0:261328:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.949436:0:261328:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572470.949439:0:261328:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572470.949443:0:261328:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572470.949446:0:261328:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572470.949448:0:261328:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.949450:0:261328:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.949454:0:261328:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.949456:0:261328:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.949459:0:261328:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.949461:0:261328:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.949463:0:261328:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572470.949464:0:261328:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572470.949466:0:261328:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572470.949469:0:261328:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d5fb58 00010000:00000001:0.0:1788572470.949471:0:261328:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572470.949472:0:261328:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572470.949474:0:261328:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572470.949476:0:261328:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.949478:0:261328:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572470.949480:0:261328:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572470.949483:0:261328:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 154 lru_type: 1 00010000:00000001:0.0:1788572470.949489:0:261328:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.949490:0:261328:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.949491:0:261328:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572470.949493:0:261328:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572470.949496:0:261328:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 155 lru_type: 1 00010000:00000001:0.0:1788572470.949503:0:261328:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572470.949506:0:261328:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572470.949509:0:261328:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.949512:0:261328:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.949513:0:261328:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.949515:0:261328:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.949518:0:261328:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572470.949519:0:261328:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.949521:0:261328:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.949524:0:261328:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.949526:0:261328:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572470.949527:0:261328:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.949528:0:261328:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572470.949531:0:261328:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572470.949533:0:261328:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572470.949534:0:261328:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572470.949539:0:261328:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572470.949543:0:261328:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572470.949545:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fae8 00000080:00002000:0.0:1788572470.949546:0:261328:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 27 00000080:00000001:0.0:1788572470.949549:0:261328:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.949551:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fb30 00000080:00200000:0.0:1788572470.949553:0:261328:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572470.949558:0:261328:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572470.949561:0:261328:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572470.949563:0:261328:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572470.949565:0:261328:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572470.949575:0:261328:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fd80. 00000020:00000001:0.0:1788572470.949579:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fae8 00000080:00200000:0.0:1788572470.949580:0:261328:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-028, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572470.949583:0:261328:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572470.949585:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f998 00000080:00200000:0.0:1788572470.949586:0:261328:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-028, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572470.949593:0:261328:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572470.949596:0:261328:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572470.949600:0:261328:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572470.949602:0:261328:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.949603:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f8b8 00800000:00000002:0.0:1788572470.949604:0:261328:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-028' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.949607:0:261328:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572470.949609:0:261328:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572470.949610:0:261328:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572470.949612:0:261328:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572470.949613:0:261328:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572470.949615:0:261328:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572470.949616:0:261328:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572470.949618:0:261328:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.949620:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f850 00800000:00000002:0.0:1788572470.949622:0:261328:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-028' -> mds #0 00000002:00000001:0.0:1788572470.949624:0:261328:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572470.949625:0:261328:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f788 00000002:00010000:0.0:1788572470.949627:0:261328:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-028,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572470.949630:0:261328:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572470.949633:0:261328:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572470.952507:0:261328:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572470.952516:0:261328:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572470.952525:0:261328:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572470.952531:0:261328:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572470.952538:0:261328:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572470.952540:0:261328:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572470.952542:0:261328:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.952545:0:261328:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572470.952548:0:261328:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572470.952553:0:261328:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572470.952561:0:261328:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572470.952570:0:261328:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.952574:0:261328:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.952576:0:261328:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.952578:0:261328:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572470.952579:0:261328:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572470.952581:0:261328:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572470.952588:0:261328:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00010000:00000001:0.0:1788572470.952593:0:261328:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572470.952596:0:261328:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572470.952603:0:261328:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572470.952606:0:261328:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572470.952609:0:261328:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572470.952612:0:261328:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b400. 00000020:00000001:0.0:1788572470.952617:0:261328:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572470.952620:0:261328:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b400 with handle 0x144158609316846f to hash 00000020:00000001:0.0:1788572470.952623:0:261328:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572470.952624:0:261328:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.952628:0:261328:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553700864 : -109714155850752 : ffff9c372eb6b400) 00010000:00000001:0.0:1788572470.952632:0:261328:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572470.953462:0:261328:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572470.953472:0:261328:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x144158609316846f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261328 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.953483:0:261328:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x144158609316846f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261328 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572470.953501:0:261328:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b400/0x144158609316846f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261328 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572470.953507:0:261328:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572470.953509:0:261328:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572470.953519:0:261328:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572470.953522:0:261328:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572470.953525:0:261328:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.953527:0:261328:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000001:0.0:1788572470.953532:0:261328:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572470.953536:0:261328:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.953538:0:261328:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572470.953540:0:261328:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572470.953552:0:261328:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440809939328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.953561:0:261328:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572470.953564:0:261328:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.953568:0:261328:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572470.953572:0:261328:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572470.953574:0:261328:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572470.953578:0:261328:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261328:1875440809939328:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572470.953582:0:261328:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572470.953588:0:261328:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572470.953591:0:261328:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572470.953594:0:261328:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572470.953598:0:261328:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec000. 02000000:00000001:0.0:1788572470.953600:0:261328:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572470.953609:0:261328:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572470.953613:0:261328:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c81b8. 00000400:00000010:0.0:1788572470.953621:0:261328:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572470.953630:0:261328:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809939328, portal 10 00000100:00000001:0.0:1788572470.953632:0:261328:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572470.953865:0:261328:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000040:0.0:1788572470.953873:0:261328:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6300 x1875440809939328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.953892:0:261328:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572470.953894:0:261328:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572470.953898:0:261328:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572470.953902:0:261328:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809939328, offset 0 00000400:00000200:0.0:1788572470.953908:0:261328:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572470.953917:0:261328:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.953924:0:261328:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572470.953930:0:261328:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572470.953934:0:261328:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572470.953937:0:261328:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572470.953940:0:261328:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572470.953945:0:261328:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539718:539718:256:4294967295] 192.168.202.148@tcp LPNI seq info [539683:539683:8:4294967295] 00000400:00000200:0.0:1788572470.953949:0:261328:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.953959:0:261328:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572470.953965:0:261328:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572470.953970:0:261328:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572470.953979:0:261328:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572470.953988:0:261328:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572470.953993:0:261328:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572470.954061:0:261328:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572470.954066:0:261328:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14983 usecs (15 jiffies) 00000400:00000200:0.0:1788572470.954070:0:261328:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572470.954072:0:261328:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307609975 00000100:00000001:0.0:1788572470.954075:0:261328:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610103 00000100:00000001:0.0:1788572470.954078:0:261328:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307609975 00000100:00000001:0.0:1788572470.954080:0:261328:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610103 00000100:00000001:0.0:1788572470.954083:0:261328:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572470.954086:0:261328:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572470.954088:0:261328:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572470.954090:0:261328:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572470.954092:0:261328:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572470.954095:0:261328:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572470.954097:0:261328:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.971483:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.971505:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.971512:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.971521:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.971529:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.971551:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.971557:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.971562:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.971566:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461180 00000400:00000200:0.0:1788572470.971571:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.971577:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ba1 [1] + 224 00000800:00000001:0.0:1788572470.971585:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.971596:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.971600:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.971604:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.971612:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.971621:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6300 x1875440809939328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.971632:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b6300 x1875440809939328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572555 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.971676:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.971685:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.971688:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572470.976082:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572470.976092:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572470.977318:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.977343:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572470.977347:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.977350:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.977359:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.977366:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572470.977373:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.977378:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.977383:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.977387:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461200 00000400:00000200:0.0:1788572470.977392:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572470.977398:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529bb1 [1] + 224 00000800:00000001:0.0:1788572470.977404:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.977411:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572470.977704:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572470.977710:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572470.977719:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572470.977728:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7480 x1875440809939456/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572470.977739:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c7480 x1875440809939456/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572486 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572470.977759:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572470.977768:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572470.977772:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572470.980781:0:261328:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572470.980790:0:261328:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.031348:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.031367:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.031371:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.031375:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.031384:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.031393:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.031404:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.031410:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.031416:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.031420:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461280 00000400:00000200:0.0:1788572471.031425:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.031431:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529bc1 [1] + 224 00000800:00000001:0.0:1788572471.031437:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.031445:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.031448:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.031453:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.031461:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.031471:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809939584/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.031482:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed69880 x1875440809939584/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.031501:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.031509:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.031512:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.044162:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.044181:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.044187:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.044189:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.044199:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.044206:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.044216:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.044222:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.044227:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.044232:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461300 00000400:00000200:0.0:1788572471.044238:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.044245:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529bc9 [1] + 224 00000800:00000001:0.0:1788572471.044253:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.044261:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.044266:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.044270:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.044280:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.044290:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809939712/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.044300:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69880 x1875440809939712/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.044333:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.044341:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.044345:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572471.045544:0:261330:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.045548:0:261330:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.045554:0:261330:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572471.045557:0:261330:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e9e0 -> ffff9c371ee2e9e0 00000020:00000001:0.0:1788572471.045559:0:261330:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e9e0 -> ffff9c371ee2e9e0 00000020:00000001:0.0:1788572471.045560:0:261330:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572471.045565:0:261330:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572471.045570:0:261330:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.045572:0:261330:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.045574:0:261330:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.045575:0:261330:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.045577:0:261330:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdb1:11031 on OST[1]: s=0 (1) m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572471.045581:0:261330:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.045583:0:261330:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572465 a=1788572465 c=1788572465 b=0 00020000:00000001:0.0:1788572471.045586:0:261330:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.045588:0:261330:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.045589:0:261330:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e9e0 -> ffff9c371ee2e9e0 00000080:00200000:0.0:1788572471.045592:0:261330:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdb1:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572471.045594:0:261330:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572471.045597:0:261330:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572471.045599:0:261330:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff53090 (ffff9c3718da63f8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572471.045602:0:261330:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.045603:0:261330:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff53090 (ffff9c3718da63f8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572471.045606:0:261330:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572471.045610:0:261330:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.045612:0:261330:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055656 : -109714416495960 : ffff9c371f2d92a8) 00000020:00000001:0.0:1788572471.045614:0:261330:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.045616:0:261330:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3720e47430 (ffff9c3718da6ea8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572471.045619:0:261330:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572471.045621:0:261330:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572471.045622:0:261330:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.045624:0:261330:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.045625:0:261330:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572471.045626:0:261330:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.045628:0:261330:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.045629:0:261330:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.045630:0:261330:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.045632:0:261330:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff53090 (ffff9c3718da63f8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572471.045688:0:261330:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572471.045690:0:261330:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.045691:0:261330:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3720e47430 (ffff9c3718da63f8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572471.045694:0:261330:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3718db31d0. 00000020:00000001:0.0:1788572471.045698:0:261330:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572471.045699:0:261330:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c3720e47400. 00020000:00000001:0.0:1788572471.045702:0:261330:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.045703:0:261330:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.045704:0:261330:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572471.045706:0:261330:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.045709:0:261330:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572471.045711:0:261330:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572471.045713:0:261330:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572471.045714:0:261330:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572471.045716:0:261330:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.045718:0:261330:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6ea0 00020000:00000001:0.0:1788572471.045721:0:261330:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572471.045723:0:261330:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572471.045725:0:261330:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572471.045727:0:261330:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdb1:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572471.045732:0:261330:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.045734:0:261330:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.045736:0:261330:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572471.045738:0:261330:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572471.045739:0:261330:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.045740:0:261330:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da63f0 00000001:00000001:0.0:1788572471.045742:0:261330:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.045782:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fd08 00000080:00000008:0.0:1788572471.045785:0:261330:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-029:[0x200002b17:0xdb1:0x0](ffff9c3719f800c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.045792:0:261330:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572471.045795:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fda0 00000080:00002000:0.0:1788572471.045797:0:261330:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-029 (ffff9c3702109d80, parent ffff9c3719d93af8, inode ffff9c3719f800c8) hashed, 00000080:00000001:0.0:1788572471.045800:0:261330:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.045845:0:261330:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.045848:0:261330:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.045850:0:261330:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.045852:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbc0 00000080:00200000:0.0:1788572471.045854:0:261330:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.045860:0:261330:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572471.045866:0:261330:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.045870:0:261330:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.045873:0:261330:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.045875:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fae0 00800000:00000002:0.0:1788572471.045888:0:261330:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.045891:0:261330:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.045895:0:261330:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.045897:0:261330:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.045899:0:261330:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.045901:0:261330:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.045903:0:261330:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.045905:0:261330:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.045907:0:261330:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.045910:0:261330:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.045916:0:261330:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.045918:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4f9b0 00000002:00010000:0.0:1788572471.045920:0:261330:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.045923:0:261330:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.045925:0:261330:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.045927:0:261330:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.045934:0:261330:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.045937:0:261330:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.045941:0:261330:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.045945:0:261330:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.045949:0:261330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 157 lru_type: 1 02000000:00000001:0.0:1788572471.045958:0:261330:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.045962:0:261330:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.045966:0:261330:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 157 lru_type: 1 00000002:00000001:0.0:1788572471.045972:0:261330:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.045974:0:261330:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.045976:0:261330:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.045978:0:261330:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.045980:0:261330:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.045982:0:261330:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.045985:0:261330:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.045988:0:261330:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.045991:0:261330:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572471.045996:0:261330:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.045998:0:261330:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.046001:0:261330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.046004:0:261330:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.046007:0:261330:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.046009:0:261330:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.046011:0:261330:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.046013:0:261330:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.046015:0:261330:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.046017:0:261330:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.046020:0:261330:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.046022:0:261330:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.046023:0:261330:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.046024:0:261330:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.046027:0:261330:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.046029:0:261330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d4fbf8 00010000:00000001:0.0:1788572471.046031:0:261330:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.046032:0:261330:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.046034:0:261330:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.046035:0:261330:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.046037:0:261330:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.046039:0:261330:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.046041:0:261330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 157 lru_type: 1 00010000:00000001:0.0:1788572471.046048:0:261330:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.046049:0:261330:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.046050:0:261330:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.046051:0:261330:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.046053:0:261330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 158 lru_type: 1 00010000:00000001:0.0:1788572471.046059:0:261330:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.046061:0:261330:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.046063:0:261330:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.046066:0:261330:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.046067:0:261330:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.046068:0:261330:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.046071:0:261330:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.046072:0:261330:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.046074:0:261330:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.046076:0:261330:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.046077:0:261330:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.046078:0:261330:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.046080:0:261330:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.046082:0:261330:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.046083:0:261330:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.046086:0:261330:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.046090:0:261330:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.046094:0:261330:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.046096:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fb88 00000080:00002000:0.0:1788572471.046097:0:261330:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 29 00000080:00000001:0.0:1788572471.046100:0:261330:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.046102:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbd0 00000080:00200000:0.0:1788572471.046104:0:261330:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572471.046108:0:261330:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.046112:0:261330:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.046113:0:261330:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.046116:0:261330:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.046118:0:261330:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.046119:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fbe8 00000080:00002000:0.0:1788572471.046121:0:261330:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-029(ffff9c3702109d80) flags 0x48000e refc 0 00000080:00000001:0.0:1788572471.046123:0:261330:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.046124:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fc30 00000080:00200000:0.0:1788572471.046126:0:261330:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-029, flags=9 00000080:00000001:0.0:1788572471.046138:0:261330:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572471.046141:0:261330:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572471.046144:0:261330:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572471.046145:0:261330:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d4fe40 00000080:00002000:0.0:1788572471.046146:0:261330:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-029 (ffff9c3702109d80, parent ffff9c3719d93af8, inode ffff9c3719f800c8) hashed, 00000080:00000001:0.0:1788572471.046149:0:261330:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572471.079527:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.079532:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572471.079546:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.079551:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.079558:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1330 00000400:00000010:0.0:1788572471.079561:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1330. 00000100:00000001:0.0:1788572471.079565:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.079570:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70e00 x1875440809939840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.079580:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.079583:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec70e00 x1875440809939840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.079590:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.079591:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572471.085045:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.085069:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.085074:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.085077:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.085087:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.085097:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.085108:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.085114:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.085119:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.085123:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461380 00000400:00000200:0.0:1788572471.085129:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.085136:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529bd1 [1] + 224 00000800:00000001:0.0:1788572471.085143:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.085150:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.085154:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.085159:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.085168:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.085177:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809939840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.085188:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec70e00 x1875440809939840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.085208:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.085219:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.085223:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572471.087515:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.087520:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572471.087526:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.087531:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.087537:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1dd0 00000400:00000010:0.0:1788572471.087539:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1dd0. 00000100:00000001:0.0:1788572471.087543:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.087546:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809939968/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.087555:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.087558:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809939968/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.087564:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.087566:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572471.096602:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.096619:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.096624:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.096627:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.096675:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.096684:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.096694:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.096699:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.096704:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.096708:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461400 00000400:00000200:0.0:1788572471.096713:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.096719:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529bd9 [1] + 224 00000800:00000001:0.0:1788572471.096728:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.096736:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.096740:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.096745:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.096755:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.096766:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809939968/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.096776:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69880 x1875440809939968/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.096791:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.096802:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.096805:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.128385:0:261334:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.128392:0:261334:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.128395:0:261334:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.128399:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fb20 00000080:00200000:0.0:1788572471.128403:0:261334:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.128412:0:261334:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572471.129039:0:261334:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.129045:0:261334:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.129050:0:261334:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.129052:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fa40 00800000:00000002:0.0:1788572471.129055:0:261334:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.129059:0:261334:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.129062:0:261334:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.129065:0:261334:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.129068:0:261334:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.129070:0:261334:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.129072:0:261334:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.129074:0:261334:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.129077:0:261334:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.129080:0:261334:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.129086:0:261334:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.129088:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f910 00000002:00010000:0.0:1788572471.129090:0:261334:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.129093:0:261334:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.129095:0:261334:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.129098:0:261334:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.129107:0:261334:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.129109:0:261334:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.129114:0:261334:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.129119:0:261334:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.129125:0:261334:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 02000000:00000001:0.0:1788572471.129137:0:261334:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.129143:0:261334:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.129146:0:261334:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00000002:00000001:0.0:1788572471.129153:0:261334:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.129155:0:261334:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.129157:0:261334:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.129159:0:261334:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.129161:0:261334:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.129164:0:261334:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.129166:0:261334:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.129169:0:261334:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.129173:0:261334:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6600. 00000080:00000001:0.0:1788572471.129181:0:261334:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.129183:0:261334:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.129186:0:261334:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.129190:0:261334:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.129193:0:261334:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.129196:0:261334:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.129197:0:261334:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.129199:0:261334:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.129202:0:261334:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.129205:0:261334:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.129207:0:261334:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.129210:0:261334:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.129212:0:261334:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.129213:0:261334:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.129215:0:261334:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.129217:0:261334:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1d5fb58 00010000:00000001:0.0:1788572471.129219:0:261334:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.129220:0:261334:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.129222:0:261334:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.129223:0:261334:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.129225:0:261334:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.129227:0:261334:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.129230:0:261334:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00010000:00000001:0.0:1788572471.129236:0:261334:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.129236:0:261334:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.129239:0:261334:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.129241:0:261334:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.129244:0:261334:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 161 lru_type: 1 00010000:00000001:0.0:1788572471.129251:0:261334:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.129254:0:261334:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.129256:0:261334:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.129260:0:261334:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.129261:0:261334:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.129263:0:261334:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129266:0:261334:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.129268:0:261334:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129270:0:261334:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129273:0:261334:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129275:0:261334:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.129276:0:261334:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.129277:0:261334:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.129280:0:261334:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.129282:0:261334:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.129285:0:261334:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.129290:0:261334:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.129293:0:261334:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.129295:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fae8 00000080:00002000:0.0:1788572471.129297:0:261334:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 30 00000080:00000001:0.0:1788572471.129300:0:261334:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.129302:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fb30 00000080:00200000:0.0:1788572471.129303:0:261334:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572471.129308:0:261334:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.129312:0:261334:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.129314:0:261334:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.129316:0:261334:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572471.129326:0:261334:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f620. 00000020:00000001:0.0:1788572471.129330:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5fae8 00000080:00200000:0.0:1788572471.129332:0:261334:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-031, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572471.129335:0:261334:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572471.129337:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f998 00000080:00200000:0.0:1788572471.129339:0:261334:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-031, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572471.129345:0:261334:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572471.129348:0:261334:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.129352:0:261334:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.129354:0:261334:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.129355:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f8b8 00800000:00000002:0.0:1788572471.129357:0:261334:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-031' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.129359:0:261334:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.129362:0:261334:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.129364:0:261334:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.129365:0:261334:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.129366:0:261334:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.129368:0:261334:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.129370:0:261334:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.129371:0:261334:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.129373:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f850 00800000:00000002:0.0:1788572471.129375:0:261334:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-031' -> mds #0 00000002:00000001:0.0:1788572471.129378:0:261334:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.129379:0:261334:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d5f788 00000002:00010000:0.0:1788572471.129381:0:261334:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-031,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572471.129385:0:261334:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572471.129388:0:261334:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572471.129393:0:261334:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572471.129397:0:261334:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572471.129403:0:261334:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.129408:0:261334:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.129410:0:261334:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.129412:0:261334:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.129413:0:261334:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.129842:0:261334:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.129845:0:261334:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.129848:0:261334:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.129856:0:261334:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572471.129865:0:261334:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129870:0:261334:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.129872:0:261334:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129873:0:261334:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.129874:0:261334:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.129887:0:261334:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.129895:0:261334:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00010000:00000001:0.0:1788572471.129898:0:261334:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.129900:0:261334:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572471.129906:0:261334:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572471.129908:0:261334:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.129911:0:261334:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.129915:0:261334:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6bc00. 00000020:00000001:0.0:1788572471.129919:0:261334:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.129921:0:261334:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6bc00 with handle 0x1441586093168499 to hash 00000020:00000001:0.0:1788572471.129996:0:261334:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.129998:0:261334:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553702912 : -109714155848704 : ffff9c372eb6bc00) 00010000:00000001:0.0:1788572471.130001:0:261334:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553702912 : -109714155848704 : ffff9c372eb6bc00) 00010000:00000001:0.0:1788572471.130004:0:261334:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.130006:0:261334:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.130010:0:261334:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6bc00/0x1441586093168499 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261334 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.130017:0:261334:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6bc00/0x1441586093168499 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261334 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.130030:0:261334:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6bc00/0x1441586093168499 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261334 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572471.130036:0:261334:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572471.130038:0:261334:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572471.130046:0:261334:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb53c0. 00000100:00000001:0.0:1788572471.130048:0:261334:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288714176 : -109714420837440 : ffff9c371eeb53c0) 00000100:00000001:0.0:1788572471.130051:0:261334:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.130053:0:261334:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000001:0.0:1788572471.130059:0:261334:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.130064:0:261334:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.130067:0:261334:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572471.130069:0:261334:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.130080:0:261334:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.130089:0:261334:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.130091:0:261334:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.130095:0:261334:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.130097:0:261334:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.130100:0:261334:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.130104:0:261334:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261334:1875440809940096:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.130108:0:261334:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.130114:0:261334:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.130116:0:261334:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.130118:0:261334:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.130123:0:261334:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ea000. 02000000:00000001:0.0:1788572471.130125:0:261334:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.130134:0:261334:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.130138:0:261334:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8268. 00000400:00000010:0.0:1788572471.130145:0:261334:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788572471.130152:0:261334:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809940096, portal 10 00000100:00000001:0.0:1788572471.130153:0:261334:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.130154:0:261334:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572471.130160:0:261334:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.130167:0:261334:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.130170:0:261334:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.130172:0:261334:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.130175:0:261334:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809940096, offset 0 00000400:00000200:0.0:1788572471.130181:0:261334:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.130190:0:261334:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.130200:0:261334:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.130206:0:261334:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.130212:0:261334:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.130216:0:261334:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.130220:0:261334:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.130224:0:261334:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539724:539724:256:4294967295] 192.168.202.148@tcp LPNI seq info [539689:539689:8:4294967295] 00000400:00000200:0.0:1788572471.130228:0:261334:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.130238:0:261334:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.130244:0:261334:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.130250:0:261334:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572471.130257:0:261334:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.130267:0:261334:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.130272:0:261334:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.130294:0:261334:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.130298:0:261334:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14988 usecs (15 jiffies) 00000400:00000200:0.0:1788572471.130302:0:261334:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.130305:0:261334:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610147 00000800:00000200:0.0:1788572471.133802:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.133813:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572471.133822:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.133833:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.133842:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.133844:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.133850:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.133859:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.133873:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.133887:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.133895:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.133897:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572471.134066:0:261334:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610262 00000100:00000001:0.0:1788572471.134072:0:261334:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610147 00000100:00000001:0.0:1788572471.134075:0:261334:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610262 00000100:00000001:0.0:1788572471.134079:0:261334:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.134082:0:261334:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.134086:0:261334:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.134088:0:261334:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572471.134091:0:261334:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb53c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572471.134095:0:261334:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.134098:0:261334:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.136035:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.136054:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.136059:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.136061:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.136071:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.136078:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.136088:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.136093:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.136098:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.136102:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461480 00000400:00000200:0.0:1788572471.136106:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.136112:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529be1 [1] + 224 00000800:00000001:0.0:1788572471.136118:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.136127:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.136130:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.136135:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.136142:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.136151:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.136162:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b5c00 x1875440809940096/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.136187:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.136197:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.136200:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.138205:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.138210:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.138213:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.138221:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.138230:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.138233:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.138237:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.138240:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.138243:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.138246:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809940224:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.138250:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.138257:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.138260:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.138262:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.138268:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572471.138270:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.138278:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.138282:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.138290:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.138297:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809940224, portal 4 00000100:00000001:0.0:1788572471.138299:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.138301:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291933824 : -109714417617792 : ffff9c371f1c7480) 00000100:00000040:0.0:1788572471.138307:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.138314:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.138317:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.138320:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572471.138323:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809940224, offset 0 00000400:00000200:0.0:1788572471.138330:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.138338:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.138345:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.138351:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.138355:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.138359:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.138362:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.138365:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539725:539725:256:4294967295] 192.168.202.148@tcp LPNI seq info [539690:539690:8:4294967295] 00000400:00000200:0.0:1788572471.138370:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.138381:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.138388:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.138394:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572471.138401:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.138408:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.138413:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.138440:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.138445:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 180740 usecs (181 jiffies) 00000400:00000200:0.0:1788572471.138448:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.138451:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610322 00000100:00000001:0.0:1788572471.138460:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307610262 00000100:00000001:0.0:1788572471.138465:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610322 00000100:00000001:0.0:1788572471.138469:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307610262 00000100:00000001:0.0:1788572471.138472:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.138474:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.138478:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.138481:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.138487:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.138489:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572471.138491:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.138492:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.138494:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.138496:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.138508:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.138510:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.148799:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.148826:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.148830:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.148834:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.148843:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.148852:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.148861:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.148865:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.148870:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.148874:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461500 00000400:00000200:0.0:1788572471.148890:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.148896:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529bf1 [1] + 224 00000800:00000001:0.0:1788572471.148905:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.148913:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.148917:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.148932:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.148939:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.148948:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.148959:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.148976:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.148985:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.148988:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.149002:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.149006:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.149012:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.149013:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.149017:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.149022:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.149025:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.149027:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.149032:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.149047:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.149052:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.149054:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.149057:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.149060:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.149063:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.149065:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.149068:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.149080:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.149085:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=8 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572471.149088:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.149090:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149093:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.149096:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.149100:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.149102:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572471.149104:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.149107:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.149112:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.149119:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.149123:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.149125:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.149128:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.149131:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149134:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149137:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=4 00000020:00000001:0.0:1788572471.149139:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149142:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149144:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149145:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149147:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=5 00000020:00000001:0.0:1788572471.149149:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149150:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149153:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.149157:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.149161:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149172:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149174:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.149176:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.149178:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149179:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149180:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=6 00000020:00000001:0.0:1788572471.149182:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149185:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000040:0.0:1788572471.149191:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c373204aa00, remote cookie: 0xd9894672d99a22ad, flags: 0x1001 00010000:00010000:0.0:1788572471.149197:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149205:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.149207:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149209:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.149213:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.149218:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.149226:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51f8:0x0].0x0 (ffff9c371fc25800) refcount = 1 00010000:00000040:0.0:1788572471.149229:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.149232:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149246:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.149247:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.149250:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.149253:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.149256:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149262:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.149263:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149264:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.149265:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149266:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.149268:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.149270:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.149273:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149274:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149275:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=5 00000020:00000001:0.0:1788572471.149278:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149280:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00000008:00000001:0.0:1788572471.149284:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.149288:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8b340 -> ffff9c3718459530 00000008:00010000:0.0:1788572471.149292:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.149300:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8b340 -> ffff9c3718459530 00000020:00000001:0.0:1788572471.149302:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.149305:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.149307:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.149310:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.149312:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.149314:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8b340 -> ffff9c3718459530 00000008:00000001:0.0:1788572471.149316:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.149318:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.149320:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.149338:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.149340:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149342:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149344:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149345:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=6 00000020:00000001:0.0:1788572471.149347:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149349:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149352:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.149355:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149361:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149363:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.149365:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149372:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.149373:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149374:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.149376:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149378:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.149379:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.149380:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c373204aa00 refcount=5 00000020:00000001:0.0:1788572471.149382:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149384:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359609141760 : -109714100409856 : ffff9c373204aa00) 00010000:00000001:0.0:1788572471.149386:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.149389:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.149395:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149396:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.149397:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.149399:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.149401:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000060c86613/0x14415860931684a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a22ad expref: -99 pid: 261334 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.149409:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.149412:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.149414:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.149418:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149419:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.149420:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149423:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.149425:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149427:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149429:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.149432:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.149433:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149434:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.149435:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.149436:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.149437:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.149443:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.149451:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809940224:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.149457:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.149460:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.149464:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440809940224/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.149471:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.149474:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.149478:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572471.149482:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.149485:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.149487:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.149489:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.149492:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215c00. 02000000:00000001:0.0:1788572471.149497:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.149499:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.149500:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788572471.149504:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.149505:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.149508:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.149518:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.149521:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.149523:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.149525:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.183421:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.183446:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.183452:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.183455:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.183468:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.183478:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.183487:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.183491:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.183497:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.183501:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461580 00000400:00000200:0.0:1788572471.183506:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.183511:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529c01 [1] + 224 00000800:00000001:0.0:1788572471.183519:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.183527:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.183531:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.183546:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.183557:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.183568:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809940352/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.183579:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6300 x1875440809940352/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.183862:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.183875:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.183892:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.197902:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.197926:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.197930:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.197934:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.197945:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.197953:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.197963:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.197969:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.197974:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.197979:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461600 00000400:00000200:0.0:1788572471.197984:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.197990:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529c11 [1] + 224 00000800:00000001:0.0:1788572471.197997:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.198005:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.198009:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.198013:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.198021:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.198030:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809940480/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.198041:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70e00 x1875440809940480/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.198062:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.198072:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.198075:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572471.204278:0:261336:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.204288:0:261336:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.244319:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.244350:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.244355:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.244358:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.244368:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.244376:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.244384:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.244388:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.244393:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.244397:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461680 00000400:00000200:0.0:1788572471.244401:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.244407:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529c19 [1] + 224 00000800:00000001:0.0:1788572471.244414:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.244430:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.244434:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.244439:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.244447:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.244459:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809940608/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.244470:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed69880 x1875440809940608/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.244496:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.244506:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.244509:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.256699:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.256718:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.256723:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.256726:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.256736:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.256744:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.256753:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.256758:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.256763:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.256768:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461700 00000400:00000200:0.0:1788572471.256773:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.256779:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529c21 [1] + 224 00000800:00000001:0.0:1788572471.256787:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.256793:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.256796:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.256800:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.256809:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.256819:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809940736/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.256830:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69880 x1875440809940736/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.256852:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.256860:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.256864:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572471.302374:0:261340:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.302380:0:261340:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:0.0:1788572471.337415:0:261342:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091c00. 00010000:00000001:0.0:1788572471.337419:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337420:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.337422:0:261342:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.337424:0:261342:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.337427:0:261342:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.337428:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.337430:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.337431:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.337433:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337436:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00000002:00010000:0.0:1788572471.337439:0:261342:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.337446:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337447:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.337449:0:261342:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.337451:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.337452:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.337453:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.337455:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337457:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000040:0.0:1788572471.337461:0:261342:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff900 count: 3 00010000:00000001:0.0:1788572471.337463:0:261342:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.337467:0:261342:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3511 (0 0) 00010000:00000001:0.0:1788572471.337470:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337471:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.337472:0:261342:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.337474:0:261342:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e8f710 00000002:00002000:0.0:1788572471.337476:0:261342:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-035 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.337479:0:261342:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.337483:0:261342:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.337486:0:261342:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.337489:0:261342:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.337492:0:261342:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.337496:0:261342:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.337499:0:261342:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.337500:0:261342:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1e8fb20 it_disposition b 00000080:00000001:0.0:1788572471.337503:0:261342:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.337509:0:261342:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.337512:0:261342:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.337515:0:261342:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329079 : 144115373145329079 : 200002b17000db7) 00000080:00000001:0.0:1788572471.337518:0:261342:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.337524:0:261342:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.337529:0:261342:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940920 : -109714522610696 : ffff9c3718da63f8) 00000020:00001000:0.0:1788572471.337531:0:261342:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da63f0 00000020:00000001:0.0:1788572471.337533:0:261342:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.337537:0:261342:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.337539:0:261342:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.337546:0:261342:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.337561:0:261342:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.337567:0:261342:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.337569:0:261342:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.337572:0:261342:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdb7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.337577:0:261342:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.337581:0:261342:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20986 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.337584:0:261342:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf1ba0) owner ffff9c371f2d0000 00020000:00000001:0.0:1788572471.337587:0:261342:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.337590:0:261342:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf1ba0) owner ffff9c371f2d0000 00020000:00000010:0.0:1788572471.337594:0:261342:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.337598:0:261342:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.337601:0:261342:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.337603:0:261342:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdb7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.337605:0:261342:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.337607:0:261342:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.337609:0:261342:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da63f0 00000080:00000001:0.0:1788572471.337614:0:261342:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329079 : 144115373145329079 : 200002b17000db7) 00000080:00000001:0.0:1788572471.337616:0:261342:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.337621:0:261342:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdb7:0x0](ffff9c3719f821c8): rc = 0 00000080:00000001:0.0:1788572471.337624:0:261342:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359205667272 : -109714503884344 : ffff9c3719f821c8) 00010000:00000001:0.0:1788572471.337626:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.337627:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.337629:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.337631:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337633:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00000080:00000001:0.0:1788572471.337669:0:261342:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.337670:0:261342:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.337672:0:261342:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940920 : -109714522610696 : ffff9c3718da63f8) 00000020:00001000:0.0:1788572471.337674:0:261342:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da63f0 00000020:00000001:0.0:1788572471.337675:0:261342:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.337677:0:261342:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.337678:0:261342:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.337681:0:261342:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.337683:0:261342:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.337686:0:261342:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.337687:0:261342:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.337690:0:261342:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdb7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.337694:0:261342:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.337697:0:261342:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20986 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.337699:0:261342:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf1ba0) owner ffff9c371f2d0000 00020000:00000001:0.0:1788572471.337701:0:261342:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.337703:0:261342:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf1ba0) owner ffff9c371f2d0000 00020000:00000010:0.0:1788572471.337706:0:261342:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.337708:0:261342:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.337710:0:261342:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.337712:0:261342:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdb7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.337714:0:261342:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.337715:0:261342:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.337718:0:261342:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.337721:0:261342:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf1ba0) owner 0000000000000000/ffff9c371f2d0000 00020000:00000002:0.0:1788572471.337724:0:261342:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ec974e0 addref 2/0 by ffff9c371f2d0000. 00020000:00000002:0.0:1788572471.337727:0:261342:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf1ba0) owner 0000000000000000/ffff9c371f2d0000 00020000:00000001:0.0:1788572471.337730:0:261342:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.337732:0:261342:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.337733:0:261342:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.337735:0:261342:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.337737:0:261342:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.337739:0:261342:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdb7:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.337741:0:261342:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da63f0 00000080:00000001:0.0:1788572471.337744:0:261342:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.337745:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337746:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.337748:0:261342:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.337751:0:261342:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.337752:0:261342:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.337755:0:261342:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdb7:0x0](ffff9c3719f821c8) for lock 0x14415860931684d1 00800000:00000001:0.0:1788572471.337758:0:261342:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.337760:0:261342:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.337761:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.337762:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.337764:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.337766:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337768:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337770:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337771:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.337772:0:261342:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.337774:0:261342:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.337780:0:261342:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37021095e8 inode ffff9c3719f821c8 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.337783:0:261342:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.337786:0:261342:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdb7:0x0](ffff9c3719f821c8) 00000080:00010000:0.0:1788572471.337789:0:261342:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684d1 from it ffffb95bc1e8fb20 00010000:00000001:0.0:1788572471.337791:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.337792:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.337794:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.337796:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337798:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.337799:0:261342:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.337803:0:261342:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.337810:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337811:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.337812:0:261342:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.337924:0:261342:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.337927:0:261342:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.337936:0:261342:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.337938:0:261342:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.337940:0:261342:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.337943:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337944:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.337946:0:261342:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.337949:0:261342:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.337951:0:261342:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.337954:0:261342:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.337957:0:261342:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.337958:0:261342:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.337960:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.337961:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.337963:0:261342:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.337967:0:261342:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3703115a00. 00000100:00000001:0.0:1788572471.337972:0:261342:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.337977:0:261342:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b100 x1875440809941120/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.337985:0:261342:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.337987:0:261342:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.337991:0:261342:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37051a4800. 02000000:00000001:0.0:1788572471.337994:0:261342:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.337997:0:261342:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.337998:0:261342:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.338001:0:261342:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.338003:0:261342:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788572471.338007:0:261342:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.338009:0:261342:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.338011:0:261342:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b100. 00000100:00000001:0.0:1788572471.338014:0:261342:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.338015:0:261342:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.338018:0:261342:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.338020:0:261342:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1e8fb20 released 00000080:00000001:0.0:1788572471.338022:0:261342:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.338033:0:261342:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e8fd08 00000080:00200008:0.0:1788572471.338036:0:261342:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-035:[0x200002b17:0xdb7:0x0](ffff9c3719f821c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.338041:0:261342:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.338043:0:261342:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e8fc80 00000080:00200000:0.0:1788572471.338044:0:261342:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdb7:0x0](ffff9c3719f821c8),name=f434.sanity-035 00000080:00000010:0.0:1788572471.338050:0:261342:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.338055:0:261342:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.338061:0:261342:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.338064:0:261342:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.338066:0:261342:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e8fba0 00800000:00000002:0.0:1788572471.338068:0:261342:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdb7:0x0] 'f434.sanity-035' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.338137:0:261342:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.338141:0:261342:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.338144:0:261342:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.338146:0:261342:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.338148:0:261342:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.338150:0:261342:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.338152:0:261342:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.338154:0:261342:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.338156:0:261342:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.338157:0:261342:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.338158:0:261342:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.338159:0:261342:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.338160:0:261342:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.338162:0:261342:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdb7:0x0] 00800000:00000001:0.0:1788572471.338163:0:261342:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.338165:0:261342:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdb7:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.338169:0:261342:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.338171:0:261342:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e8fa70 00000002:00010000:0.0:1788572471.338172:0:261342:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdb7:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.338176:0:261342:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.338178:0:261342:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.338179:0:261342:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.338184:0:261342:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff900 count: 3 00010000:00000001:0.0:1788572471.338186:0:261342:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.338188:0:261342:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.338191:0:261342:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.338194:0:261342:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.338201:0:261342:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.338205:0:261342:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.338208:0:261342:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.338218:0:261342:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.338220:0:261342:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.338222:0:261342:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.338223:0:261342:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.338225:0:261342:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.338226:0:261342:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.338228:0:261342:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.338230:0:261342:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.338232:0:261342:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.338238:0:261342:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.338239:0:261342:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.338241:0:261342:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdb7:0x0](ffff9c3719f821c8) 00000080:00010000:0.0:1788572471.338244:0:261342:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdb7:0x0](ffff9c3719f821c8) for lock 0x14415860931684d1 00800000:00000001:0.0:1788572471.338246:0:261342:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.338248:0:261342:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.338249:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.338250:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.338252:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.338253:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.338255:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.338257:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.338258:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.338260:0:261342:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.338261:0:261342:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.338263:0:261342:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684d1 from it ffffb95bc1e8fcb8 00010000:00000001:0.0:1788572471.338264:0:261342:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.338265:0:261342:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.338266:0:261342:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c7200 refcount=3 00000020:00000001:0.0:1788572471.338268:0:261342:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.338270:0:261342:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710315008 : -109713999236608 : ffff9c37380c7200) 00010000:00000001:0.0:1788572471.338272:0:261342:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.338274:0:261342:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.338280:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.338281:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.338282:0:261342:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.338283:0:261342:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.338285:0:261342:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c7200/0x14415860931684d1 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdb7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2332 expref: -99 pid: 261342 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.338291:0:261342:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.338292:0:261342:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.338293:0:261342:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.338295:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.338296:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.338297:0:261342:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338299:0:261342:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.338300:0:261342:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338302:0:261342:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338304:0:261342:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338305:0:261342:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.338307:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.338308:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.338309:0:261342:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.338311:0:261342:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.338314:0:261342:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.338316:0:261342:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.338318:0:261342:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.338320:0:261342:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.338321:0:261342:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.338323:0:261342:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.338326:0:261342:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.338328:0:261342:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.338330:0:261342:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940920 : -109714522610696 : ffff9c3718da63f8) 00000020:00001000:0.0:1788572471.338332:0:261342:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da63f0 00000020:00000001:0.0:1788572471.338336:0:261342:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e080 -> ffff9c3720b2e080 00000020:00000001:0.0:1788572471.338338:0:261342:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.338340:0:261342:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.338343:0:261342:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdb7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.338345:0:261342:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.338347:0:261342:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.338348:0:261342:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.338350:0:261342:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.338351:0:261342:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.338355:0:261342:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdb7:0x0]io ffff9c371ff530f0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.338358:0:261342:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf1ba0) owner 0000000000000000/ffff9c371f2d0000 00020000:00000001:0.0:1788572471.338361:0:261342:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.338362:0:261342:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.338364:0:261342:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.338365:0:261342:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.338367:0:261342:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.338369:0:261342:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.338371:0:261342:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.338372:0:261342:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.338373:0:261342:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.338374:0:261342:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.338376:0:261342:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf1ba0) owner 0000000000000000/ffff9c371f2d0000 00000020:00000001:0.0:1788572471.338378:0:261342:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.338381:0:261342:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.338382:0:261342:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdb7:0x0] 00000020:00000001:0.0:1788572471.338384:0:261342:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.338386:0:261342:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.338388:0:261342:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.338390:0:261342:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.338395:0:261342:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.338400:0:261342:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676480. 00020000:00000002:0.0:1788572471.338402:0:261342:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.338405:0:261342:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.338407:0:261342:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.338408:0:261342:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.338410:0:261342:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.338411:0:261342:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943656 : -109714522607960 : ffff9c3718da6ea8) 00000020:00001000:0.0:1788572471.338413:0:261342:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6ea0 00000020:00000001:0.0:1788572471.338416:0:261342:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e9e0 -> ffff9c3720b2e080 00000020:00000001:0.0:1788572471.338417:0:261342:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.338420:0:261342:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.338422:0:261342:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.338423:0:261342:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.338424:0:261342:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.338426:0:261342:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055656 : -109714416495960 : ffff9c371f2d92a8) 00000020:00000001:0.0:1788572471.338427:0:261342:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.338431:0:261342:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6f70. 00000008:00010000:0.0:1788572471.338433:0:261342:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.338435:0:261342:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.338436:0:261342:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.338438:0:261342:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d6764b0, osc lock ffff9c37030a6f70, flags 0x400001002 00000020:00000001:0.0:1788572471.338440:0:261342:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.338442:0:261342:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.338444:0:261342:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725952 : -109714177825664 : ffff9c372d676480) 00000020:00000001:0.0:1788572471.338446:0:261342:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.338447:0:261342:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.338448:0:261342:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.338450:0:261342:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.338452:0:261342:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.338454:0:261342:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.338456:0:261342:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.338458:0:261342:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.338459:0:261342:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.338460:0:261342:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055656 : -109714416495960 : ffff9c371f2d92a8) 00000020:00000001:0.0:1788572471.338462:0:261342:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.338464:0:261342:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.338466:0:261342:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.338469:0:261342:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.338472:0:261342:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.338473:0:261342:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.338474:0:261342:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.338476:0:261342:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.338477:0:261342:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.338479:0:261342:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.338480:0:261342:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.338485:0:261342:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338488:0:261342:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.338492:0:261342:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.338496:0:261342:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.338501:0:261342:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.338505:0:261342:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.338509:0:261342:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.338510:0:261342:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.338511:0:261342:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.338514:0:261342:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.338516:0:261342:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.338518:0:261342:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.338523:0:261342:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.338529:0:261342:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338533:0:261342:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.338535:0:261342:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338536:0:261342:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.338537:0:261342:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338539:0:261342:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.338542:0:261342:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.338545:0:261342:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.338547:0:261342:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.338551:0:261342:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76d00. 00010000:00000010:0.0:1788572471.338554:0:261342:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3708c76900. 00010000:00000001:0.0:1788572471.338557:0:261342:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.338561:0:261342:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6aa00. 00000020:00000001:0.0:1788572471.338565:0:261342:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.338567:0:261342:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6aa00 with handle 0x14415860931684d8 to hash 00000020:00000001:0.0:1788572471.338569:0:261342:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.338571:0:261342:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000010:0.0:1788572471.338574:0:261342:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091480. 00010000:00000001:0.0:1788572471.338576:0:261342:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.338579:0:261342:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.338581:0:261342:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.338584:0:261342:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.338596:0:261342:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.338607:0:261342:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.338611:0:261342:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.338615:0:261342:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.338622:0:261342:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.339243:0:261342:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.339248:0:261342:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.339250:0:261342:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.339252:0:261342:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.339254:0:261342:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.339257:0:261342:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.339258:0:261342:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.339261:0:261342:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.339262:0:261342:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.339272:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.339274:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.344532:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.344537:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.344588:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.344591:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.344594:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2f20. 00000400:00000200:0.0:1788572471.344598:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3cc0 00000400:00000010:0.0:1788572471.344600:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3cc0. 00000100:00000001:0.0:1788572471.344603:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.344608:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.344616:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.344621:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.344623:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.344626:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.344629:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.344632:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.344721:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.344725:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.344736:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.344740:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=8 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572471.344743:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.344744:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.344747:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.344751:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.344754:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.344756:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572471.344758:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.344761:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.344766:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.344774:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.344777:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.344779:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.344782:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.344784:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.344787:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.344791:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=4 00000020:00000001:0.0:1788572471.344793:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.344796:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.344799:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.344800:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.344801:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=5 00000020:00000001:0.0:1788572471.344803:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.344805:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.344809:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.344812:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.344890:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.344908:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.344910:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.344912:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.344915:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.344916:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.344918:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=6 00000020:00000001:0.0:1788572471.344920:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.344922:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000040:0.0:1788572471.344930:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6aa00, remote cookie: 0xd9894672d99a2339, flags: 0x1001 00010000:00010000:0.0:1788572471.344936:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.344944:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.344947:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.344949:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.344953:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.344957:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.344966:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51fa:0x0].0x0 (ffff9c3708c76d00) refcount = 1 00010000:00000040:0.0:1788572471.344969:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.344972:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.344981:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.344983:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.344986:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.344988:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.344992:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.344998:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.345000:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345001:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.345002:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345003:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.345005:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.345007:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.345010:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.345011:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.345012:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=5 00000020:00000001:0.0:1788572471.345014:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.345016:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00000008:00000001:0.0:1788572471.345020:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.345025:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac1c0 -> ffff9c3720b2e080 00000008:00010000:0.0:1788572471.345029:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.345037:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac1c0 -> ffff9c3720b2e080 00000020:00000001:0.0:1788572471.345039:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.345042:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.345045:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.345048:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.345049:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.345051:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac1c0 -> ffff9c3720b2e080 00000008:00000001:0.0:1788572471.345053:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.345056:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.345058:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.345173:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.345176:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345178:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.345180:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.345182:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=6 00000020:00000001:0.0:1788572471.345184:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.345186:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.345189:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.345192:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.345199:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345200:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.345203:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.345210:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.345211:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345213:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.345214:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345216:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.345217:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.345218:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6aa00 refcount=5 00000020:00000001:0.0:1788572471.345220:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.345222:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698304 : -109714155853312 : ffff9c372eb6aa00) 00010000:00000001:0.0:1788572471.345223:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.345226:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.345232:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345233:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.345235:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.345238:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.345240:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 000000005de79b27/0x14415860931684d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a2339 expref: -99 pid: 261342 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.345249:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.345253:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.345255:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.345258:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345259:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.345261:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345264:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.345266:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345268:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345270:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.345271:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.345273:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345274:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.345275:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.345276:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.345277:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.345283:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.345293:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809941248:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.345300:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.345302:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.345305:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809941248/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.345312:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.345314:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.345318:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe96400. 02000000:00000001:0.0:1788572471.345323:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.345325:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.345327:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.345329:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.345332:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572471.345338:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.345340:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.345342:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.345345:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.345346:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.345349:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.345355:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.345357:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.345359:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.345361:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.374796:0:261344:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.374804:0:261344:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.374806:0:261344:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.374810:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7b20 00000080:00200000:0.0:1788572471.374815:0:261344:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.374823:0:261344:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.374830:0:261344:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.374834:0:261344:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.374840:0:261344:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.374842:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7a40 00800000:00000002:0.0:1788572471.374844:0:261344:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.374848:0:261344:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.374851:0:261344:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.374853:0:261344:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.374856:0:261344:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.374858:0:261344:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.374860:0:261344:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.374862:0:261344:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.374864:0:261344:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.374867:0:261344:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.374873:0:261344:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.374874:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7910 00000002:00010000:0.0:1788572471.374876:0:261344:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.378397:0:261344:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.378401:0:261344:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.378405:0:261344:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.378413:0:261344:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.378469:0:261344:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.378475:0:261344:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.378480:0:261344:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.378485:0:261344:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 170 lru_type: 1 02000000:00000001:0.0:1788572471.378497:0:261344:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.378502:0:261344:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.378506:0:261344:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 170 lru_type: 1 00000002:00000001:0.0:1788572471.378512:0:261344:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.378515:0:261344:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.378517:0:261344:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.378520:0:261344:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.378522:0:261344:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.378525:0:261344:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.378528:0:261344:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.378532:0:261344:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.378536:0:261344:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.378543:0:261344:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.378545:0:261344:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.378548:0:261344:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.378552:0:261344:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.378556:0:261344:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.378559:0:261344:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.378561:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.378563:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.378566:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.378567:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.378570:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.378573:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.378574:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.378575:0:261344:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.378577:0:261344:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.378579:0:261344:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1ed7b58 00010000:00000001:0.0:1788572471.378582:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.378582:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.378584:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.378586:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.378588:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.378590:0:261344:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.378593:0:261344:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 170 lru_type: 1 00010000:00000001:0.0:1788572471.378600:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.378601:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.378602:0:261344:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.378604:0:261344:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.378607:0:261344:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 171 lru_type: 1 00010000:00000001:0.0:1788572471.378614:0:261344:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.378617:0:261344:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.378619:0:261344:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.378622:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.378624:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.378625:0:261344:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.378628:0:261344:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.378630:0:261344:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.378631:0:261344:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.378903:0:261344:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.378907:0:261344:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.378908:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.378909:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.378912:0:261344:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.378914:0:261344:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.378917:0:261344:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.378923:0:261344:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.378928:0:261344:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.378931:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7ae8 00000080:00002000:0.0:1788572471.378933:0:261344:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 35 00000080:00000001:0.0:1788572471.378936:0:261344:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.379066:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7b30 00000080:00200000:0.0:1788572471.379068:0:261344:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572471.379142:0:261344:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.379149:0:261344:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.379151:0:261344:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.379155:0:261344:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572471.379166:0:261344:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f000. 00000020:00000001:0.0:1788572471.379171:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7ae8 00000080:00200000:0.0:1788572471.379173:0:261344:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-036, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572471.379176:0:261344:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572471.379178:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7998 00000080:00200000:0.0:1788572471.379180:0:261344:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-036, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572471.379188:0:261344:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.379193:0:261344:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.379197:0:261344:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.379200:0:261344:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.379202:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed78b8 00800000:00000002:0.0:1788572471.379204:0:261344:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-036' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.379207:0:261344:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.379210:0:261344:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.379213:0:261344:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.379215:0:261344:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.379218:0:261344:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.379220:0:261344:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.379222:0:261344:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.379224:0:261344:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.379227:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7850 00800000:00000002:0.0:1788572471.379229:0:261344:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-036' -> mds #0 00000002:00000001:0.0:1788572471.379232:0:261344:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.379234:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7788 00000002:00010000:0.0:1788572471.379236:0:261344:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-036,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572471.379240:0:261344:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572471.379243:0:261344:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572471.379247:0:261344:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.379251:0:261344:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572471.379258:0:261344:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.379262:0:261344:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.379265:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.379266:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.379268:0:261344:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.379271:0:261344:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.379273:0:261344:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.379276:0:261344:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.379283:0:261344:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572471.379289:0:261344:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.379293:0:261344:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.379295:0:261344:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.379297:0:261344:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.379298:0:261344:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.379301:0:261344:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.379307:0:261344:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.379311:0:261344:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.379313:0:261344:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572471.379318:0:261344:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572471.379320:0:261344:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.379322:0:261344:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.379326:0:261344:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6ac00. 00000020:00000001:0.0:1788572471.379330:0:261344:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.379332:0:261344:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6ac00 with handle 0x14415860931684df to hash 00000020:00000001:0.0:1788572471.379334:0:261344:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.379335:0:261344:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.379339:0:261344:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.379341:0:261344:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.379343:0:261344:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.379347:0:261344:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261344 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.379354:0:261344:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261344 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.379367:0:261344:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261344 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572471.379373:0:261344:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572471.379374:0:261344:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572471.379381:0:261344:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.379383:0:261344:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572471.379385:0:261344:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.379387:0:261344:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572471.379392:0:261344:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.379396:0:261344:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.379398:0:261344:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572471.379399:0:261344:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.379410:0:261344:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.379476:0:261344:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.379478:0:261344:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.379482:0:261344:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.379484:0:261344:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.379487:0:261344:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.379490:0:261344:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261344:1875440809941376:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.379495:0:261344:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.379500:0:261344:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.379503:0:261344:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.379505:0:261344:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.379510:0:261344:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.379512:0:261344:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.379519:0:261344:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.379522:0:261344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.379529:0:261344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.379536:0:261344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809941376, portal 10 00000100:00000001:0.0:1788572471.379538:0:261344:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.379540:0:261344:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.379545:0:261344:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.379552:0:261344:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.379554:0:261344:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.379556:0:261344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572471.379559:0:261344:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809941376, offset 0 00000400:00000200:0.0:1788572471.379565:0:261344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.379573:0:261344:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.379579:0:261344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.379584:0:261344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.379587:0:261344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.379591:0:261344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.379593:0:261344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.379597:0:261344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539734:539734:256:4294967295] 192.168.202.148@tcp LPNI seq info [539699:539699:8:4294967295] 00000400:00000200:0.0:1788572471.379601:0:261344:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.379609:0:261344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.379614:0:261344:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.379619:0:261344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572471.379625:0:261344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.379632:0:261344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.379914:0:261344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.379968:0:261344:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.379974:0:261344:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14985 usecs (15 jiffies) 00000400:00000200:0.0:1788572471.379979:0:261344:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.379982:0:261344:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610401 00000100:00000001:0.0:1788572471.379985:0:261344:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610526 00000100:00000001:0.0:1788572471.379988:0:261344:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610401 00000100:00000001:0.0:1788572471.379990:0:261344:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610526 00000100:00000001:0.0:1788572471.379993:0:261344:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.379996:0:261344:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.379998:0:261344:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.380000:0:261344:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572471.380003:0:261344:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572471.380005:0:261344:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.380008:0:261344:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.387543:0:261344:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.387559:0:261344:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.387561:0:261344:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.387565:0:261344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.387569:0:261344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.387571:0:261344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.387574:0:261344:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.387579:0:261344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.387588:0:261344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.387594:0:261344:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.387596:0:261344:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.387599:0:261344:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.387602:0:261344:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.387605:0:261344:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.387608:0:261344:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.387611:0:261344:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.387620:0:261344:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.387625:0:261344:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.387627:0:261344:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.387629:0:261344:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.387632:0:261344:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.387667:0:261344:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.387673:0:261344:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.387675:0:261344:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.387677:0:261344:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.387679:0:261344:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.387685:0:261344:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.387691:0:261344:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.387694:0:261344:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.387696:0:261344:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.387699:0:261344:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.387706:0:261344:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261344:1875440809941376:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.387712:0:261344:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.387715:0:261344:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.387717:0:261344:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.387719:0:261344:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.387722:0:261344:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.387729:0:261344:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.387731:0:261344:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.387735:0:261344:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.387736:0:261344:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.387740:0:261344:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.387744:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.387746:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.387749:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=4 00000020:00000001:0.0:1788572471.387751:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387829:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000040:0.0:1788572471.387838:0:261344:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6ac00, remote cookie: 0xd9894672d99a2355, flags: 0x1001 00010000:00010000:0.0:1788572471.387843:0:261344:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.387853:0:261344:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdb8:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.387856:0:261344:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.387864:0:261344:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff800 count: 2 00010000:00000001:0.0:1788572471.387867:0:261344:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.387870:0:261344:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.387873:0:261344:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.387891:0:261344:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.387898:0:261344:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.387902:0:261344:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.387904:0:261344:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.387906:0:261344:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.387907:0:261344:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.387909:0:261344:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdb8:0x0].0x0 (ffff9c3713fff800) refcount = 2 00010000:00000040:0.0:1788572471.387912:0:261344:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.387914:0:261344:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c370166a400/0x1441586093167ebf lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a0f74 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.387921:0:261344:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.387928:0:261344:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.387932:0:261344:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.387934:0:261344:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.387937:0:261344:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.387939:0:261344:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.387943:0:261344:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.387948:0:261344:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.387950:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.387951:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.387953:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.387954:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.387956:0:261344:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.387960:0:261344:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.387962:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.387963:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.387965:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.387967:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387969:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387971:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.387972:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.387973:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=4 00000020:00000001:0.0:1788572471.387975:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387977:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387979:0:261344:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.387981:0:261344:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.387985:0:261344:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.387991:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.387992:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.387993:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.387994:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.387996:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=5 00000020:00000001:0.0:1788572471.387997:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.387999:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388002:0:261344:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.388005:0:261344:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.388011:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388012:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.388015:0:261344:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.388021:0:261344:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.388022:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388023:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388024:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388025:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.388030:0:261344:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.388039:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388040:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388042:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388044:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388045:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00000002:00010000:0.0:1788572471.388049:0:261344:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.388058:0:261344:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091e40. 00010000:00000001:0.0:1788572471.388061:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388062:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.388063:0:261344:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.388065:0:261344:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.388068:0:261344:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.388071:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388072:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388074:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388075:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388077:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00000002:00010000:0.0:1788572471.388081:0:261344:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.388087:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388088:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388090:0:261344:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.388091:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388093:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388094:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388095:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388097:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000040:0.0:1788572471.388100:0:261344:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff800 count: 3 00010000:00000001:0.0:1788572471.388101:0:261344:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.388105:0:261344:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3512 (0 0) 00010000:00000001:0.0:1788572471.388107:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388109:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.388110:0:261344:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.388112:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7710 00000002:00002000:0.0:1788572471.388114:0:261344:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-036 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.388117:0:261344:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.388121:0:261344:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.388124:0:261344:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.388127:0:261344:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.388130:0:261344:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.388133:0:261344:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.388136:0:261344:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.388137:0:261344:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1ed7b20 it_disposition b 00000080:00000001:0.0:1788572471.388141:0:261344:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.388146:0:261344:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.388149:0:261344:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.388152:0:261344:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329080 : 144115373145329080 : 200002b17000db8) 00000080:00000001:0.0:1788572471.388155:0:261344:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.388161:0:261344:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.388166:0:261344:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.388168:0:261344:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.388171:0:261344:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.388174:0:261344:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.388176:0:261344:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.388182:0:261344:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.388186:0:261344:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.388192:0:261344:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.388194:0:261344:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.388198:0:261344:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdb8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.388202:0:261344:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.388206:0:261344:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21101 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.388210:0:261344:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3730ca3000) owner ffff9c37012a0000 00020000:00000001:0.0:1788572471.388213:0:261344:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.388215:0:261344:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3730ca3000) owner ffff9c37012a0000 00020000:00000010:0.0:1788572471.388219:0:261344:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.388222:0:261344:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.388225:0:261344:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.388227:0:261344:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdb8:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.388229:0:261344:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.388231:0:261344:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.388233:0:261344:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572471.388236:0:261344:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329080 : 144115373145329080 : 200002b17000db8) 00000080:00000001:0.0:1788572471.388239:0:261344:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.388243:0:261344:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdb8:0x0](ffff9c3708701988): rc = 0 00000080:00000001:0.0:1788572471.388245:0:261344:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911539592 : -109714798012024 : ffff9c3708701988) 00010000:00000001:0.0:1788572471.388247:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388249:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388250:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388251:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388254:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00000080:00000001:0.0:1788572471.388256:0:261344:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.388258:0:261344:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.388259:0:261344:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.388260:0:261344:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.388262:0:261344:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.388263:0:261344:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.388264:0:261344:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.388266:0:261344:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.388269:0:261344:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.388271:0:261344:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.388272:0:261344:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.388274:0:261344:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdb8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.388278:0:261344:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.388280:0:261344:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21101 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.388283:0:261344:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3730ca3000) owner ffff9c37012a0000 00020000:00000001:0.0:1788572471.388285:0:261344:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.388286:0:261344:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3730ca3000) owner ffff9c37012a0000 00020000:00000010:0.0:1788572471.388288:0:261344:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.388290:0:261344:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.388292:0:261344:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.388294:0:261344:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdb8:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.388295:0:261344:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.388297:0:261344:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.388299:0:261344:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.388303:0:261344:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca3000) owner 0000000000000000/ffff9c37012a0000 00020000:00000002:0.0:1788572471.388306:0:261344:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c370438e480 addref 2/0 by ffff9c37012a0000. 00020000:00000002:0.0:1788572471.388308:0:261344:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca3000) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572471.388311:0:261344:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.388313:0:261344:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.388314:0:261344:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.388316:0:261344:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.388318:0:261344:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.388320:0:261344:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdb8:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.388322:0:261344:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572471.388324:0:261344:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388325:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388327:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.388328:0:261344:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.388331:0:261344:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.388332:0:261344:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.388335:0:261344:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdb8:0x0](ffff9c3708701988) for lock 0x14415860931684df 00800000:00000001:0.0:1788572471.388338:0:261344:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.388340:0:261344:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.388341:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388342:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388343:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388345:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388347:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388349:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388350:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.388351:0:261344:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.388353:0:261344:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.388358:0:261344:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f9bd0 inode ffff9c3708701988 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.388361:0:261344:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.388365:0:261344:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdb8:0x0](ffff9c3708701988) 00000080:00010000:0.0:1788572471.388369:0:261344:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684df from it ffffb95bc1ed7b20 00010000:00000001:0.0:1788572471.388371:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388371:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388373:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388374:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388376:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388378:0:261344:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.388381:0:261344:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.388388:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388390:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388391:0:261344:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.388393:0:261344:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.388395:0:261344:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.388403:0:261344:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.388405:0:261344:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.388408:0:261344:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.388411:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388412:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388414:0:261344:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388417:0:261344:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.388419:0:261344:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388421:0:261344:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388423:0:261344:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388425:0:261344:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.388426:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388427:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.388429:0:261344:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.388432:0:261344:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.388438:0:261344:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.388443:0:261344:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809941376/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.388450:0:261344:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.388453:0:261344:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.388457:0:261344:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.388460:0:261344:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.388463:0:261344:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.388464:0:261344:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.388466:0:261344:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.388469:0:261344:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572471.388473:0:261344:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.388474:0:261344:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.388476:0:261344:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.388479:0:261344:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.388481:0:261344:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.388483:0:261344:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.388484:0:261344:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1ed7b20 released 00000080:00000001:0.0:1788572471.388486:0:261344:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.388497:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7d08 00000080:00200008:0.0:1788572471.388499:0:261344:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-036:[0x200002b17:0xdb8:0x0](ffff9c3708701988), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.388504:0:261344:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.388506:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7c80 00000080:00200000:0.0:1788572471.388507:0:261344:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdb8:0x0](ffff9c3708701988),name=f434.sanity-036 00000080:00000010:0.0:1788572471.388512:0:261344:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.388518:0:261344:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.388523:0:261344:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.388526:0:261344:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.388528:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7ba0 00800000:00000002:0.0:1788572471.388529:0:261344:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdb8:0x0] 'f434.sanity-036' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.388533:0:261344:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.388536:0:261344:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.388539:0:261344:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.388541:0:261344:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.388543:0:261344:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.388545:0:261344:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.388547:0:261344:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.388549:0:261344:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.388551:0:261344:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.388552:0:261344:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.388553:0:261344:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.388554:0:261344:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.388555:0:261344:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.388557:0:261344:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdb8:0x0] 00800000:00000001:0.0:1788572471.388558:0:261344:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.388560:0:261344:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdb8:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.388564:0:261344:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.388566:0:261344:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7a70 00000002:00010000:0.0:1788572471.388568:0:261344:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdb8:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.388571:0:261344:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.388573:0:261344:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.388574:0:261344:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.388578:0:261344:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff800 count: 3 00010000:00000001:0.0:1788572471.388580:0:261344:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.388583:0:261344:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.388585:0:261344:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.388588:0:261344:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.388601:0:261344:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.388605:0:261344:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.388608:0:261344:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.388614:0:261344:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.388616:0:261344:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.388617:0:261344:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.388619:0:261344:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.388620:0:261344:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.388622:0:261344:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.388624:0:261344:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.388626:0:261344:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.388628:0:261344:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.388631:0:261344:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.388632:0:261344:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.388661:0:261344:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdb8:0x0](ffff9c3708701988) 00000080:00010000:0.0:1788572471.388664:0:261344:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdb8:0x0](ffff9c3708701988) for lock 0x14415860931684df 00800000:00000001:0.0:1788572471.388666:0:261344:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.388668:0:261344:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.388669:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388670:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388672:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388673:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388675:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388677:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388678:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.388679:0:261344:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.388680:0:261344:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.388682:0:261344:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684df from it ffffb95bc1ed7cb8 00010000:00000001:0.0:1788572471.388684:0:261344:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.388685:0:261344:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.388686:0:261344:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ac00 refcount=3 00000020:00000001:0.0:1788572471.388688:0:261344:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388690:0:261344:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553698816 : -109714155852800 : ffff9c372eb6ac00) 00010000:00000001:0.0:1788572471.388691:0:261344:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.388693:0:261344:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.388698:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388699:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388700:0:261344:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.388701:0:261344:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.388703:0:261344:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6ac00/0x14415860931684df lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2355 expref: -99 pid: 261344 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.388708:0:261344:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.388710:0:261344:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.388711:0:261344:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.388713:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388714:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.388715:0:261344:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388717:0:261344:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.388718:0:261344:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388719:0:261344:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388721:0:261344:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.388723:0:261344:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.388724:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.388725:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.388726:0:261344:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.388728:0:261344:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.388730:0:261344:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.388733:0:261344:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.388735:0:261344:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.388737:0:261344:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.388738:0:261344:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.388740:0:261344:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.388743:0:261344:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.388745:0:261344:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.388747:0:261344:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.388749:0:261344:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.388753:0:261344:0:(cl_object.c:143:cl_object_top()) ffff9c3718459148 -> ffff9c3718459148 00000020:00000001:0.0:1788572471.390618:0:261344:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.390625:0:261344:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.390628:0:261344:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdb8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.390633:0:261344:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.390689:0:261344:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.390693:0:261344:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.390695:0:261344:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.390697:0:261344:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.390702:0:261344:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdb8:0x0]io ffff9c371ff50bf0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.390706:0:261344:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca3000) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572471.390711:0:261344:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.390713:0:261344:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.390715:0:261344:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.390716:0:261344:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.390719:0:261344:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.390721:0:261344:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.390722:0:261344:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.390723:0:261344:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.390724:0:261344:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.390725:0:261344:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.390726:0:261344:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca3000) owner 0000000000000000/ffff9c37012a0000 00000020:00000001:0.0:1788572471.390729:0:261344:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.390732:0:261344:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.390733:0:261344:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdb8:0x0] 00000020:00000001:0.0:1788572471.390735:0:261344:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.390737:0:261344:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.390739:0:261344:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.390741:0:261344:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.390746:0:261344:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.390754:0:261344:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676900. 00020000:00000002:0.0:1788572471.390757:0:261344:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.390759:0:261344:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.390898:0:261344:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.390900:0:261344:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.390903:0:261344:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.390906:0:261344:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572471.390909:0:261344:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572471.390913:0:261344:0:(cl_object.c:143:cl_object_top()) ffff9c3718459468 -> ffff9c3718459148 00000020:00000001:0.0:1788572471.390914:0:261344:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.390918:0:261344:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.390919:0:261344:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.390921:0:261344:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.390922:0:261344:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.390924:0:261344:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572471.390926:0:261344:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.390932:0:261344:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6130. 00000008:00010000:0.0:1788572471.390936:0:261344:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.390939:0:261344:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.390940:0:261344:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.390941:0:261344:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676930, osc lock ffff9c37030a6130, flags 0x400001002 00000020:00000001:0.0:1788572471.390943:0:261344:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.390945:0:261344:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.390947:0:261344:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531727104 : -109714177824512 : ffff9c372d676900) 00000020:00000001:0.0:1788572471.390949:0:261344:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.390950:0:261344:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.390952:0:261344:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.390953:0:261344:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.390956:0:261344:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.390957:0:261344:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.390959:0:261344:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.390960:0:261344:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.390962:0:261344:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.390963:0:261344:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572471.390964:0:261344:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.390967:0:261344:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.390968:0:261344:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.390971:0:261344:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.390974:0:261344:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.390975:0:261344:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.390977:0:261344:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.390978:0:261344:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.390980:0:261344:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.390983:0:261344:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.390986:0:261344:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.390992:0:261344:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.390995:0:261344:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.391001:0:261344:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.391005:0:261344:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.391010:0:261344:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.391015:0:261344:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.391020:0:261344:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.391022:0:261344:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.391023:0:261344:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.391026:0:261344:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.391029:0:261344:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.391033:0:261344:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.391038:0:261344:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.391045:0:261344:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.391049:0:261344:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.391051:0:261344:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.391052:0:261344:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.391054:0:261344:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.391056:0:261344:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.391058:0:261344:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.391061:0:261344:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.391063:0:261344:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.391067:0:261344:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76c00. 00010000:00000010:0.0:1788572471.391070:0:261344:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3708c76300. 00010000:00000001:0.0:1788572471.391623:0:261344:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.391630:0:261344:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6ba00. 00000020:00000001:0.0:1788572471.391663:0:261344:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.391666:0:261344:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6ba00 with handle 0x14415860931684e6 to hash 00000020:00000001:0.0:1788572471.391668:0:261344:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.391670:0:261344:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000010:0.0:1788572471.391675:0:261344:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091400. 00010000:00000001:0.0:1788572471.391678:0:261344:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.391683:0:261344:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.391687:0:261344:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.391691:0:261344:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.391704:0:261344:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.391716:0:261344:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.391722:0:261344:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.391725:0:261344:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.391737:0:261344:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.392250:0:261344:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.392253:0:261344:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.392255:0:261344:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.392257:0:261344:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.392259:0:261344:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.392262:0:261344:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.392264:0:261344:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.392265:0:261344:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.392267:0:261344:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.392275:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.392278:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572471.396200:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.396210:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572471.396219:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.396227:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.396235:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788572471.396238:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788572471.396243:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.396250:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.396262:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.396266:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.396274:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.396276:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572471.401709:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.401713:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.401720:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.401721:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.401725:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2528. 00000400:00000200:0.0:1788572471.401730:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f35d8 00000400:00000010:0.0:1788572471.401732:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f35d8. 00000100:00000001:0.0:1788572471.401735:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.401739:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.401748:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.401755:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.401757:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.401759:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.401762:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.401764:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.401766:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.401769:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.401777:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.401781:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=86 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572471.401783:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.401785:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.401788:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.401791:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.401794:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.401796:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572471.401798:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.401801:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.401806:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.401826:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.401829:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.401831:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.401834:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.401836:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.401839:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.401842:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=4 00000020:00000001:0.0:1788572471.401844:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401847:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401849:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.401850:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.401851:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=5 00000020:00000001:0.0:1788572471.401853:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401855:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401858:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.401860:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.401865:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.401875:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.401877:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.401886:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.401888:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.401889:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.401890:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=6 00000020:00000001:0.0:1788572471.401892:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401894:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000040:0.0:1788572471.401900:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6ba00, remote cookie: 0xd9894672d99a235c, flags: 0x1001 00010000:00010000:0.0:1788572471.401905:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.401919:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.401921:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.401924:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.401927:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.401931:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.401939:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x526d:0x0].0x0 (ffff9c3708c76c00) refcount = 1 00010000:00000040:0.0:1788572471.401941:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.401944:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.401952:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.401954:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.401956:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.401959:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.401962:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.401968:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.401969:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.401971:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.401972:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.401973:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.401975:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.401977:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.401980:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.401981:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.401983:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=5 00000020:00000001:0.0:1788572471.401984:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.401986:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00000008:00000001:0.0:1788572471.401991:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.401995:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a540 -> ffff9c3718459148 00000008:00010000:0.0:1788572471.402000:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.402007:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a540 -> ffff9c3718459148 00000020:00000001:0.0:1788572471.402008:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.402011:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.402014:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.402016:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.402017:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.402019:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8a540 -> ffff9c3718459148 00000008:00000001:0.0:1788572471.402021:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.402023:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.402025:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.402105:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.402108:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402110:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.402112:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.402113:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=6 00000020:00000001:0.0:1788572471.402115:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.402118:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.402120:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.402123:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.402131:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402132:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.402134:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.402141:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.402142:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402143:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.402145:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402147:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.402148:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.402149:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ba00 refcount=5 00000020:00000001:0.0:1788572471.402151:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.402152:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553702400 : -109714155849216 : ffff9c372eb6ba00) 00010000:00000001:0.0:1788572471.402154:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.402156:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.402162:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402163:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.402164:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.402166:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.402169:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000070f8aafe/0x14415860931684e6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x526d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a235c expref: -99 pid: 261344 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.402176:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.402178:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.402180:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.402184:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402185:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.402186:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402189:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.402191:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402193:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402195:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.402197:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.402198:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402199:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.402200:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.402201:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.402202:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.402208:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.402217:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809941504:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.402222:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.402225:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.402228:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809941504/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.402235:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.402237:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.402241:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe97800. 02000000:00000001:0.0:1788572471.402245:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.402248:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.402249:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.402251:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.402254:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572471.402259:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.402260:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.402262:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.402266:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.402267:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.402269:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.402276:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.402277:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.402280:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.402282:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453721:0:261346:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453725:0:261346:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.453732:0:261346:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572471.453735:0:261346:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e850 -> ffff9c3720b2e850 00000020:00000001:0.0:1788572471.453737:0:261346:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e850 -> ffff9c3720b2e850 00000020:00000001:0.0:1788572471.453739:0:261346:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572471.453743:0:261346:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572471.453747:0:261346:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.453749:0:261346:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.453751:0:261346:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.453752:0:261346:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.453769:0:261346:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdb9:11031 on OST[1]: s=0 (1) m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572471.453772:0:261346:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.453774:0:261346:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572471.453776:0:261346:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453778:0:261346:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453780:0:261346:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e850 -> ffff9c3720b2e850 00000080:00200000:0.0:1788572471.453782:0:261346:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdb9:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572471.453785:0:261346:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572471.453788:0:261346:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572471.453790:0:261346:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572471.453792:0:261346:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.453794:0:261346:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572471.453797:0:261346:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572471.453801:0:261346:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.453803:0:261346:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572471.453805:0:261346:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.453806:0:261346:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cff30 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572471.453809:0:261346:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572471.453810:0:261346:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572471.453812:0:261346:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.453814:0:261346:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.453822:0:261346:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572471.453823:0:261346:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.453825:0:261346:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.453826:0:261346:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.453828:0:261346:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.453829:0:261346:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572471.453832:0:261346:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572471.453833:0:261346:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.453834:0:261346:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cff30 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572471.453837:0:261346:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb9300. 00000020:00000001:0.0:1788572471.453840:0:261346:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572471.453842:0:261346:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cff00. 00020000:00000001:0.0:1788572471.453846:0:261346:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.453847:0:261346:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.453848:0:261346:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572471.453851:0:261346:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453854:0:261346:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572471.453856:0:261346:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572471.453858:0:261346:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572471.453859:0:261346:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572471.453861:0:261346:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.453863:0:261346:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572471.453866:0:261346:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572471.453868:0:261346:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572471.453871:0:261346:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572471.453873:0:261346:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdb9:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572471.453877:0:261346:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.453890:0:261346:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.453892:0:261346:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572471.453894:0:261346:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572471.453895:0:261346:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.453896:0:261346:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572471.453899:0:261346:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.453906:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7d08 00000080:00000008:0.0:1788572471.453909:0:261346:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-037:[0x200002b17:0xdb9:0x0](ffff9c3719f81148), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.453915:0:261346:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572471.453917:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7da0 00000080:00002000:0.0:1788572471.453919:0:261346:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-037 (ffff9c3719d93bd0, parent ffff9c3719d93af8, inode ffff9c3719f81148) hashed, 00000080:00000001:0.0:1788572471.453921:0:261346:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.453952:0:261346:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.453956:0:261346:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.453958:0:261346:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.453959:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7bc0 00000080:00200000:0.0:1788572471.453962:0:261346:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.453968:0:261346:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.453975:0:261346:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.453979:0:261346:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.453983:0:261346:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.453986:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7ae0 00800000:00000002:0.0:1788572471.453988:0:261346:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.453991:0:261346:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.453994:0:261346:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.453996:0:261346:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.453999:0:261346:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.454003:0:261346:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.454005:0:261346:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.454009:0:261346:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.454011:0:261346:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.454015:0:261346:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.454019:0:261346:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.454021:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed79b0 00000002:00010000:0.0:1788572471.454023:0:261346:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.454026:0:261346:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.454028:0:261346:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.454031:0:261346:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.454038:0:261346:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.454041:0:261346:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.454045:0:261346:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.454049:0:261346:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.454053:0:261346:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 173 lru_type: 1 02000000:00000001:0.0:1788572471.454061:0:261346:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.454066:0:261346:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.454069:0:261346:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 173 lru_type: 1 00000002:00000001:0.0:1788572471.454075:0:261346:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.454077:0:261346:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.454079:0:261346:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.454080:0:261346:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.454082:0:261346:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.454084:0:261346:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.454087:0:261346:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.454090:0:261346:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.454093:0:261346:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.454098:0:261346:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.454099:0:261346:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.454102:0:261346:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.454106:0:261346:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.454122:0:261346:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.454124:0:261346:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.454125:0:261346:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.454126:0:261346:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.454128:0:261346:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.454130:0:261346:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.454132:0:261346:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.454135:0:261346:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.454136:0:261346:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.454137:0:261346:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.454139:0:261346:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.454141:0:261346:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1ed7bf8 00010000:00000001:0.0:1788572471.454143:0:261346:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.454144:0:261346:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.454145:0:261346:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.454147:0:261346:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.454148:0:261346:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.454150:0:261346:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.454153:0:261346:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 173 lru_type: 1 00010000:00000001:0.0:1788572471.454159:0:261346:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.454160:0:261346:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.454161:0:261346:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.454162:0:261346:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.454165:0:261346:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 174 lru_type: 1 00010000:00000001:0.0:1788572471.454171:0:261346:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.454173:0:261346:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.454175:0:261346:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.454178:0:261346:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.454179:0:261346:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.454180:0:261346:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.454182:0:261346:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.454184:0:261346:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.454185:0:261346:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.454188:0:261346:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.454189:0:261346:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.454190:0:261346:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.454191:0:261346:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.454193:0:261346:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.454195:0:261346:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.454197:0:261346:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.454201:0:261346:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.454205:0:261346:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.454207:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7b88 00000080:00002000:0.0:1788572471.454209:0:261346:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 37 00000080:00000001:0.0:1788572471.454211:0:261346:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.454213:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7bd0 00000080:00200000:0.0:1788572471.454214:0:261346:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572471.454220:0:261346:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.454223:0:261346:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.454225:0:261346:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.454227:0:261346:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.454230:0:261346:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.454231:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7be8 00000080:00002000:0.0:1788572471.454233:0:261346:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-037(ffff9c3719d93bd0) flags 0x48000e refc 0 00000080:00000001:0.0:1788572471.454235:0:261346:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.454236:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7c30 00000080:00200000:0.0:1788572471.454238:0:261346:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-037, flags=9 00000080:00000001:0.0:1788572471.454249:0:261346:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572471.454252:0:261346:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572471.454254:0:261346:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572471.454255:0:261346:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1ed7e40 00000080:00002000:0.0:1788572471.454257:0:261346:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-037 (ffff9c3719d93bd0, parent ffff9c3719d93af8, inode ffff9c3719f81148) hashed, 00000080:00000001:0.0:1788572471.454259:0:261346:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.474413:0:261348:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.474421:0:261348:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.474423:0:261348:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.474427:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7b20 00000080:00200000:0.0:1788572471.474432:0:261348:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.474440:0:261348:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.474447:0:261348:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.474451:0:261348:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.474456:0:261348:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.474458:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7a40 00800000:00000002:0.0:1788572471.474461:0:261348:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.474465:0:261348:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.474467:0:261348:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.474470:0:261348:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.474472:0:261348:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.474475:0:261348:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.474477:0:261348:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.474479:0:261348:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.474481:0:261348:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.474484:0:261348:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.478994:0:261348:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.479005:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7910 00000002:00010000:0.0:1788572471.479010:0:261348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.479015:0:261348:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.479017:0:261348:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.479021:0:261348:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.479030:0:261348:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.479033:0:261348:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.479038:0:261348:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.479045:0:261348:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.479050:0:261348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 174 lru_type: 1 02000000:00000001:0.0:1788572471.479062:0:261348:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.479069:0:261348:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.479950:0:261348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 174 lru_type: 1 00000002:00000001:0.0:1788572471.479964:0:261348:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.479967:0:261348:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.479969:0:261348:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.479972:0:261348:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.479975:0:261348:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.479977:0:261348:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.479981:0:261348:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.479985:0:261348:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.479989:0:261348:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.479997:0:261348:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.479999:0:261348:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.480002:0:261348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.480006:0:261348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.480010:0:261348:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.480012:0:261348:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.480014:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.480017:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.480020:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.480022:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.480025:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.480028:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.480029:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.480030:0:261348:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.480032:0:261348:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.480035:0:261348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc1eb7b58 00010000:00000001:0.0:1788572471.480037:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.480038:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.480039:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.480041:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.480043:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.480045:0:261348:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.480049:0:261348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 174 lru_type: 1 00010000:00000001:0.0:1788572471.480056:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.480057:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.480059:0:261348:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.480061:0:261348:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.480063:0:261348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 175 lru_type: 1 00010000:00000001:0.0:1788572471.480071:0:261348:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.480727:0:261348:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.480731:0:261348:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.480736:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.480738:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.480739:0:261348:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.480743:0:261348:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.480745:0:261348:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.480747:0:261348:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.480751:0:261348:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.480753:0:261348:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.481154:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.481155:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.481159:0:261348:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.481160:0:261348:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.481164:0:261348:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.481172:0:261348:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.481178:0:261348:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.481182:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7ae8 00000080:00002000:0.0:1788572471.481184:0:261348:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 37 00000080:00000001:0.0:1788572471.481187:0:261348:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.481189:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7b30 00000080:00200000:0.0:1788572471.481191:0:261348:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572471.481196:0:261348:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.481202:0:261348:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.481204:0:261348:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.481207:0:261348:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572471.481218:0:261348:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fa80. 00000020:00000001:0.0:1788572471.481222:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7ae8 00000080:00200000:0.0:1788572471.481224:0:261348:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-038, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572471.481227:0:261348:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572471.481229:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7998 00000080:00200000:0.0:1788572471.481231:0:261348:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-038, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572471.481240:0:261348:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.481244:0:261348:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.481249:0:261348:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.481253:0:261348:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.481254:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb78b8 00800000:00000002:0.0:1788572471.481256:0:261348:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-038' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.481259:0:261348:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.481263:0:261348:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.481266:0:261348:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.481268:0:261348:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.481270:0:261348:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.481272:0:261348:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.481274:0:261348:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.481277:0:261348:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.481280:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7850 00800000:00000002:0.0:1788572471.481281:0:261348:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-038' -> mds #0 00000002:00000001:0.0:1788572471.481285:0:261348:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.481287:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7788 00000002:00010000:0.0:1788572471.481289:0:261348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-038,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572471.481293:0:261348:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572471.481295:0:261348:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572471.481301:0:261348:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.481305:0:261348:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572471.481311:0:261348:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.481315:0:261348:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.481331:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.481333:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.481334:0:261348:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.481338:0:261348:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.481341:0:261348:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.481345:0:261348:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.481352:0:261348:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572471.481360:0:261348:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.481366:0:261348:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.481367:0:261348:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.481368:0:261348:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.481370:0:261348:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.481371:0:261348:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.481379:0:261348:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.481383:0:261348:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.481386:0:261348:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572471.481392:0:261348:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572471.481395:0:261348:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.481397:0:261348:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.481401:0:261348:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6a200. 00000020:00000001:0.0:1788572471.481405:0:261348:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.481408:0:261348:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6a200 with handle 0x14415860931684fb to hash 00000020:00000001:0.0:1788572471.481410:0:261348:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.481412:0:261348:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.481415:0:261348:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.481419:0:261348:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.481421:0:261348:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.481425:0:261348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261348 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.481434:0:261348:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261348 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.481444:0:261348:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261348 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572471.481449:0:261348:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572471.481451:0:261348:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572471.481458:0:261348:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.481460:0:261348:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572471.481463:0:261348:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.481464:0:261348:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572471.481469:0:261348:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.481473:0:261348:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.481475:0:261348:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572471.481477:0:261348:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.481487:0:261348:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.481495:0:261348:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.481497:0:261348:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.481500:0:261348:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.481502:0:261348:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.481504:0:261348:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.481508:0:261348:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261348:1875440809941888:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.481512:0:261348:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.481516:0:261348:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.481519:0:261348:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.481521:0:261348:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.481526:0:261348:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.481527:0:261348:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.481534:0:261348:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.481538:0:261348:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.481545:0:261348:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.481552:0:261348:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809941888, portal 10 00000100:00000001:0.0:1788572471.481553:0:261348:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.481554:0:261348:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.481560:0:261348:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.481566:0:261348:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.481569:0:261348:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.481572:0:261348:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572471.481574:0:261348:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809941888, offset 0 00000400:00000200:0.0:1788572471.481580:0:261348:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.481588:0:261348:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.481595:0:261348:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.481600:0:261348:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.481604:0:261348:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.481607:0:261348:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.481610:0:261348:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.481613:0:261348:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539738:539738:256:4294967295] 192.168.202.148@tcp LPNI seq info [539703:539703:8:4294967295] 00000400:00000200:0.0:1788572471.481618:0:261348:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.481627:0:261348:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.481633:0:261348:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.483925:0:261348:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788572471.483938:0:261348:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.483946:0:261348:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.483950:0:261348:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0e00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.484017:0:261348:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.484024:0:261348:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14981 usecs (15 jiffies) 00000400:00000200:0.0:1788572471.484030:0:261348:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.484034:0:261348:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610505 00000100:00000001:0.0:1788572471.484037:0:261348:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307610634 00000100:00000001:0.0:1788572471.484039:0:261348:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610505 00000100:00000001:0.0:1788572471.484041:0:261348:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307610634 00000100:00000001:0.0:1788572471.484044:0:261348:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.484048:0:261348:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.484050:0:261348:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.484052:0:261348:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572471.484055:0:261348:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572471.484058:0:261348:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.484061:0:261348:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.498736:0:261348:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.498745:0:261348:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.498747:0:261348:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.498751:0:261348:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.498766:0:261348:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.498768:0:261348:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.498771:0:261348:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.498776:0:261348:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.498785:0:261348:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.498791:0:261348:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.498793:0:261348:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.498795:0:261348:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.498799:0:261348:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.498801:0:261348:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.498804:0:261348:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.498807:0:261348:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.498826:0:261348:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.498831:0:261348:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.498834:0:261348:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.498836:0:261348:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.498839:0:261348:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.498843:0:261348:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.498847:0:261348:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.498849:0:261348:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.498851:0:261348:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.498855:0:261348:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.498860:0:261348:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.498866:0:261348:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.498869:0:261348:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.498871:0:261348:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.498874:0:261348:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.498892:0:261348:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261348:1875440809941888:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.498897:0:261348:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.498900:0:261348:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.498902:0:261348:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.498904:0:261348:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.498907:0:261348:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.498912:0:261348:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.498915:0:261348:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.498918:0:261348:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.498919:0:261348:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.498922:0:261348:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.498926:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.498928:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.498932:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=4 00000020:00000001:0.0:1788572471.498934:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.498946:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000040:0.0:1788572471.498954:0:261348:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6a200, remote cookie: 0xd9894672d99a239b, flags: 0x1001 00010000:00010000:0.0:1788572471.498959:0:261348:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.498967:0:261348:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdba:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.498970:0:261348:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.498976:0:261348:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff200 count: 2 00010000:00000001:0.0:1788572471.498979:0:261348:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.498982:0:261348:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.498985:0:261348:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.498997:0:261348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499002:0:261348:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.499006:0:261348:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.499008:0:261348:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.499010:0:261348:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.499011:0:261348:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.499014:0:261348:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdba:0x0].0x0 (ffff9c3713fff200) refcount = 2 00010000:00000040:0.0:1788572471.499016:0:261348:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.499019:0:261348:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c370166a200/0x1441586093167edb lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdba:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a0fd6 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.499027:0:261348:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499033:0:261348:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.499037:0:261348:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.499039:0:261348:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.499041:0:261348:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.499044:0:261348:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.499047:0:261348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499053:0:261348:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.499054:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499056:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499057:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499057:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499059:0:261348:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.499063:0:261348:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.499064:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499066:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499067:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499069:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499071:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499085:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499086:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499087:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=4 00000020:00000001:0.0:1788572471.499089:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499091:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499094:0:261348:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.499097:0:261348:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.499100:0:261348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499107:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499108:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499109:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499110:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499111:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=5 00000020:00000001:0.0:1788572471.499112:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499114:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499116:0:261348:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.499118:0:261348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499124:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499125:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.499128:0:261348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499134:0:261348:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.499136:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499137:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499138:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499139:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.499144:0:261348:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.499153:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499154:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499156:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499158:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499160:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00000002:00010000:0.0:1788572471.499163:0:261348:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.499173:0:261348:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091640. 00010000:00000001:0.0:1788572471.499175:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499176:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.499178:0:261348:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.499180:0:261348:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.499183:0:261348:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.499185:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499186:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499188:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499190:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499192:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00000002:00010000:0.0:1788572471.499195:0:261348:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499200:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499201:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499203:0:261348:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.499204:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499205:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499206:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499208:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499210:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000040:0.0:1788572471.499213:0:261348:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff200 count: 3 00010000:00000001:0.0:1788572471.499214:0:261348:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.499218:0:261348:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3514 (0 0) 00010000:00000001:0.0:1788572471.499221:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499222:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.499223:0:261348:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.499226:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7710 00000002:00002000:0.0:1788572471.499228:0:261348:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-038 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.499231:0:261348:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.499235:0:261348:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.499238:0:261348:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.499242:0:261348:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.499244:0:261348:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.499247:0:261348:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.499251:0:261348:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.499252:0:261348:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1eb7b20 it_disposition b 00000080:00000001:0.0:1788572471.499255:0:261348:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.499261:0:261348:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.499263:0:261348:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.499266:0:261348:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329082 : 144115373145329082 : 200002b17000dba) 00000080:00000001:0.0:1788572471.499269:0:261348:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.499275:0:261348:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.499279:0:261348:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.499281:0:261348:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.499284:0:261348:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.499287:0:261348:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.499290:0:261348:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.499296:0:261348:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.499299:0:261348:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.499305:0:261348:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.499307:0:261348:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.499311:0:261348:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdba:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.499316:0:261348:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.499320:0:261348:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21102 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.499324:0:261348:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf15d0) owner ffff9c3711b04f80 00020000:00000001:0.0:1788572471.499328:0:261348:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.499330:0:261348:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf15d0) owner ffff9c3711b04f80 00020000:00000010:0.0:1788572471.499336:0:261348:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.499339:0:261348:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.499342:0:261348:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.499343:0:261348:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdba:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.499346:0:261348:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.499347:0:261348:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.499349:0:261348:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572471.499353:0:261348:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329082 : 144115373145329082 : 200002b17000dba) 00000080:00000001:0.0:1788572471.499355:0:261348:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.499360:0:261348:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdba:0x0](ffff9c3719f87448): rc = 0 00000080:00000001:0.0:1788572471.499362:0:261348:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359205688392 : -109714503863224 : ffff9c3719f87448) 00010000:00000001:0.0:1788572471.499365:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499366:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499367:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499369:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499371:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00000080:00000001:0.0:1788572471.499375:0:261348:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.499376:0:261348:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.499377:0:261348:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.499379:0:261348:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.499380:0:261348:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.499381:0:261348:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.499382:0:261348:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.499384:0:261348:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.499386:0:261348:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.499388:0:261348:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.499390:0:261348:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.499393:0:261348:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdba:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.499396:0:261348:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.499400:0:261348:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21102 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.499402:0:261348:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf15d0) owner ffff9c3711b04f80 00020000:00000001:0.0:1788572471.499404:0:261348:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.499405:0:261348:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf15d0) owner ffff9c3711b04f80 00020000:00000010:0.0:1788572471.499408:0:261348:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.499410:0:261348:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.499411:0:261348:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.499413:0:261348:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdba:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.499415:0:261348:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.499416:0:261348:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.499419:0:261348:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.499422:0:261348:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf15d0) owner 0000000000000000/ffff9c3711b04f80 00020000:00000002:0.0:1788572471.499425:0:261348:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ec97480 addref 2/0 by ffff9c3711b04f80. 00020000:00000002:0.0:1788572471.499427:0:261348:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf15d0) owner 0000000000000000/ffff9c3711b04f80 00020000:00000001:0.0:1788572471.499430:0:261348:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.499432:0:261348:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.499433:0:261348:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.499434:0:261348:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.499437:0:261348:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.499439:0:261348:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdba:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.499441:0:261348:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572471.499443:0:261348:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499445:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499446:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.499448:0:261348:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.499450:0:261348:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.499452:0:261348:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.499455:0:261348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdba:0x0](ffff9c3719f87448) for lock 0x14415860931684fb 00800000:00000001:0.0:1788572471.499457:0:261348:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.499459:0:261348:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.499460:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499461:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499463:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499464:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499466:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499469:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499470:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.499471:0:261348:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.499473:0:261348:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.499479:0:261348:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f9510 inode ffff9c3719f87448 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.499483:0:261348:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.499486:0:261348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdba:0x0](ffff9c3719f87448) 00000080:00010000:0.0:1788572471.499490:0:261348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684fb from it ffffb95bc1eb7b20 00010000:00000001:0.0:1788572471.499492:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499493:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499494:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499496:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499498:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499500:0:261348:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.499503:0:261348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.499508:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499509:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499511:0:261348:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.499513:0:261348:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.499516:0:261348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.499523:0:261348:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.499525:0:261348:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.499527:0:261348:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.499531:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499532:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499534:0:261348:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499536:0:261348:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.499538:0:261348:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499540:0:261348:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499542:0:261348:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499544:0:261348:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.499546:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499547:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.499548:0:261348:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.499551:0:261348:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.499557:0:261348:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.499561:0:261348:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809941888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.499568:0:261348:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.499570:0:261348:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.499573:0:261348:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.499576:0:261348:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.499578:0:261348:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.499580:0:261348:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.499582:0:261348:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.499584:0:261348:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572471.499587:0:261348:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.499589:0:261348:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.499591:0:261348:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.499595:0:261348:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.499596:0:261348:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.499598:0:261348:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.499600:0:261348:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1eb7b20 released 00000080:00000001:0.0:1788572471.499601:0:261348:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.499612:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7d08 00000080:00200008:0.0:1788572471.499614:0:261348:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-038:[0x200002b17:0xdba:0x0](ffff9c3719f87448), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.499619:0:261348:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.499620:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7c80 00000080:00200000:0.0:1788572471.499622:0:261348:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdba:0x0](ffff9c3719f87448),name=f434.sanity-038 00000080:00000010:0.0:1788572471.499627:0:261348:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.499631:0:261348:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.499694:0:261348:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.499697:0:261348:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.499699:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7ba0 00800000:00000002:0.0:1788572471.499701:0:261348:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdba:0x0] 'f434.sanity-038' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.499704:0:261348:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.499708:0:261348:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.499710:0:261348:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.499713:0:261348:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.499715:0:261348:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.499717:0:261348:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.499719:0:261348:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.499721:0:261348:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.499723:0:261348:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.499724:0:261348:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.499726:0:261348:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.499727:0:261348:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.499728:0:261348:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.499730:0:261348:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdba:0x0] 00800000:00000001:0.0:1788572471.499731:0:261348:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.499733:0:261348:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdba:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.499737:0:261348:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.499738:0:261348:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eb7a70 00000002:00010000:0.0:1788572471.499740:0:261348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdba:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.499743:0:261348:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.499746:0:261348:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.499747:0:261348:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.499750:0:261348:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff200 count: 3 00010000:00000001:0.0:1788572471.499752:0:261348:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.499754:0:261348:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.499756:0:261348:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.499760:0:261348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.499766:0:261348:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.499770:0:261348:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.499774:0:261348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.499780:0:261348:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.499782:0:261348:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.499784:0:261348:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.499786:0:261348:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.499787:0:261348:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.499789:0:261348:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.499791:0:261348:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.499792:0:261348:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.499794:0:261348:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.499798:0:261348:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.499799:0:261348:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.499801:0:261348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdba:0x0](ffff9c3719f87448) 00000080:00010000:0.0:1788572471.499804:0:261348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdba:0x0](ffff9c3719f87448) for lock 0x14415860931684fb 00800000:00000001:0.0:1788572471.499806:0:261348:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.499807:0:261348:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.499808:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499809:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499811:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499813:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499815:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499817:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499819:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.499820:0:261348:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.499821:0:261348:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.499823:0:261348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931684fb from it ffffb95bc1eb7cb8 00010000:00000001:0.0:1788572471.499824:0:261348:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.499825:0:261348:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.499827:0:261348:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a200 refcount=3 00000020:00000001:0.0:1788572471.499828:0:261348:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499830:0:261348:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696256 : -109714155855360 : ffff9c372eb6a200) 00010000:00000001:0.0:1788572471.499832:0:261348:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.499834:0:261348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.499840:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499841:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499843:0:261348:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.499844:0:261348:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.499846:0:261348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a200/0x14415860931684fb lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a239b expref: -99 pid: 261348 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.499852:0:261348:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.499854:0:261348:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.499855:0:261348:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.499857:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499858:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.499859:0:261348:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499861:0:261348:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.499862:0:261348:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499864:0:261348:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499866:0:261348:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.499867:0:261348:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.499868:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.499869:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.499871:0:261348:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.499872:0:261348:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.499875:0:261348:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.499877:0:261348:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.499891:0:261348:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.499893:0:261348:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.499894:0:261348:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.499896:0:261348:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.499899:0:261348:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.499901:0:261348:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.499903:0:261348:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572471.499905:0:261348:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572471.499910:0:261348:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e788 -> ffff9c3720b2e788 00000020:00000001:0.0:1788572471.499912:0:261348:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.499915:0:261348:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.499917:0:261348:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.499920:0:261348:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.499921:0:261348:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.499923:0:261348:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.499925:0:261348:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.499927:0:261348:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.499930:0:261348:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdba:0x0]io ffff9c371ff50bf0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.499932:0:261348:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf15d0) owner 0000000000000000/ffff9c3711b04f80 00020000:00000001:0.0:1788572471.499935:0:261348:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.499937:0:261348:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.499939:0:261348:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.499940:0:261348:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.499942:0:261348:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.499944:0:261348:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.499946:0:261348:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.499947:0:261348:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.499948:0:261348:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.499949:0:261348:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.499950:0:261348:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf15d0) owner 0000000000000000/ffff9c3711b04f80 00000020:00000001:0.0:1788572471.499952:0:261348:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.499954:0:261348:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.499956:0:261348:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdba:0x0] 00000020:00000001:0.0:1788572471.499958:0:261348:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.499960:0:261348:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.499962:0:261348:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.499964:0:261348:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.499968:0:261348:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.499973:0:261348:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676a00. 00020000:00000002:0.0:1788572471.499975:0:261348:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.499977:0:261348:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.499979:0:261348:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.499980:0:261348:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.499981:0:261348:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.499983:0:261348:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572471.499985:0:261348:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572471.499987:0:261348:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e5f8 -> ffff9c3720b2e788 00000020:00000001:0.0:1788572471.499988:0:261348:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.499992:0:261348:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.499993:0:261348:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.499994:0:261348:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.499996:0:261348:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.499998:0:261348:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572471.500000:0:261348:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.500004:0:261348:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a7690. 00000008:00010000:0.0:1788572471.500007:0:261348:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.500009:0:261348:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.500011:0:261348:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.500012:0:261348:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676a30, osc lock ffff9c37030a7690, flags 0x400001002 00000020:00000001:0.0:1788572471.500015:0:261348:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.500016:0:261348:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.500019:0:261348:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531727360 : -109714177824256 : ffff9c372d676a00) 00000020:00000001:0.0:1788572471.500020:0:261348:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.500021:0:261348:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.500023:0:261348:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.500025:0:261348:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.500027:0:261348:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.500029:0:261348:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.500030:0:261348:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.500031:0:261348:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.500032:0:261348:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.500034:0:261348:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572471.500035:0:261348:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.500038:0:261348:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.500039:0:261348:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.500043:0:261348:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.500045:0:261348:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.500046:0:261348:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.500047:0:261348:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.500049:0:261348:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.500050:0:261348:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.500052:0:261348:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.500053:0:261348:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.500056:0:261348:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.500059:0:261348:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.500062:0:261348:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.500065:0:261348:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.500069:0:261348:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.500072:0:261348:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.500075:0:261348:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.500076:0:261348:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.500078:0:261348:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.500081:0:261348:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.500083:0:261348:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.500086:0:261348:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.500090:0:261348:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.500096:0:261348:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.500099:0:261348:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.500101:0:261348:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.500102:0:261348:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.500103:0:261348:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.500105:0:261348:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.500107:0:261348:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.500110:0:261348:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.500112:0:261348:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.500116:0:261348:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3708c76700. 00010000:00000010:0.0:1788572471.500120:0:261348:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3707e6ad00. 00010000:00000001:0.0:1788572471.500123:0:261348:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.500125:0:261348:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6a400. 00000020:00000001:0.0:1788572471.500128:0:261348:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.500130:0:261348:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6a400 with handle 0x1441586093168502 to hash 00000020:00000001:0.0:1788572471.500132:0:261348:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.500133:0:261348:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000010:0.0:1788572471.500136:0:261348:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091380. 00010000:00000001:0.0:1788572471.500138:0:261348:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.500140:0:261348:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.500142:0:261348:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.500145:0:261348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.500154:0:261348:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.500162:0:261348:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.500166:0:261348:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.500170:0:261348:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.500176:0:261348:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.500198:0:261348:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.500201:0:261348:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.500203:0:261348:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.500204:0:261348:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.500206:0:261348:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.500208:0:261348:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.500209:0:261348:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.500211:0:261348:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.500213:0:261348:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.500225:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572471.500229:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572471.500231:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.500237:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.500239:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.500241:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.500243:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.500245:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.500250:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.500257:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.500259:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.500262:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.500264:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.500266:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.500270:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809942016:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.500274:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.500279:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.500281:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.500283:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.500287:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572471.500289:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.500296:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.500300:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.500305:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.500311:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809942016, portal 4 00000100:00000001:0.0:1788572471.500313:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.500315:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.500320:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.500327:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.500330:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.500332:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788572471.500335:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809942016, offset 0 00000400:00000200:0.0:1788572471.500340:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.500347:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.500353:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.500358:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.500362:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.500365:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.500368:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.500372:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539739:539739:256:4294967295] 192.168.202.148@tcp LPNI seq info [539704:539704:8:4294967295] 00000400:00000200:0.0:1788572471.500376:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.500383:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.500389:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.500394:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572471.500400:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.500407:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.500411:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.500419:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.500424:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 170061 usecs (171 jiffies) 00000400:00000200:0.0:1788572471.500427:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.500430:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610676 00000100:00000001:0.0:1788572471.500438:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307610634 00000100:00000001:0.0:1788572471.500443:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610676 00000100:00000001:0.0:1788572471.500446:0:2403:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307610634 00000100:00000001:0.0:1788572471.500449:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.500451:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.500454:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.500456:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.509179:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.509184:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.509190:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.509192:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.509195:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.509200:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.509203:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.509206:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.509211:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.509219:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.509226:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.509228:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.509230:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.509233:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.509235:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.509237:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.509240:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.509249:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.509253:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=86 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572471.509255:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.509257:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509261:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.509264:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.509267:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.509269:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572471.509271:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.509274:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.509279:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.509286:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.509289:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.509291:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.509294:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.509296:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509299:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509302:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=4 00000020:00000001:0.0:1788572471.509304:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509307:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509310:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509310:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509312:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=5 00000020:00000001:0.0:1788572471.509314:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509315:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509319:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.509322:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.509326:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509338:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509340:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.509342:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.509344:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509345:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509347:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=6 00000020:00000001:0.0:1788572471.509349:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509361:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000040:0.0:1788572471.509368:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6a400, remote cookie: 0xd9894672d99a23a2, flags: 0x1001 00010000:00010000:0.0:1788572471.509373:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509381:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.509384:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509387:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.509390:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.509395:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.509403:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x526e:0x0].0x0 (ffff9c3708c76700) refcount = 1 00010000:00000040:0.0:1788572471.509406:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.509409:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509423:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.509425:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.509429:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.509431:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.509434:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509441:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.509442:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509444:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.509445:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509446:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.509447:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.509450:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.509453:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509454:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509455:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=5 00000020:00000001:0.0:1788572471.509458:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509459:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00000008:00000001:0.0:1788572471.509463:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.509468:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad500 -> ffff9c3720b2e788 00000008:00010000:0.0:1788572471.509472:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.509479:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad500 -> ffff9c3720b2e788 00000020:00000001:0.0:1788572471.509481:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.509484:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.509487:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.509489:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.509491:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.509493:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad500 -> ffff9c3720b2e788 00000008:00000001:0.0:1788572471.509495:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.509497:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.509499:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.509572:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.509575:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509577:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509579:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509580:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=6 00000020:00000001:0.0:1788572471.509582:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509585:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509587:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.509590:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509597:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509598:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.509601:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509607:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.509608:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509609:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.509610:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509612:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.509613:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.509614:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a400 refcount=5 00000020:00000001:0.0:1788572471.509616:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509618:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553696768 : -109714155854848 : ffff9c372eb6a400) 00010000:00000001:0.0:1788572471.509620:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.509622:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.509628:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509629:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.509630:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.509633:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.509660:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000018bc0395/0x1441586093168502 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x526e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a23a2 expref: -99 pid: 261348 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.509668:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.509671:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.509673:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.509676:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509677:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.509679:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509682:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.509684:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509686:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509689:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.509690:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.509691:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509692:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.509693:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.509694:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.509696:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.509702:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.509711:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809942016:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.509717:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.509720:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.509724:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809942016/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.509731:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.509734:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.509737:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572471.509741:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.509743:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.509745:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.509747:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.509749:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572471.509755:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.509757:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.509758:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.509772:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.509774:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.509776:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.509783:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.509784:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.509786:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.509788:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.576766:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.576791:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.576795:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.576798:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.576807:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.576816:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.576828:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.576833:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.576838:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.576843:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461c80 00000400:00000200:0.0:1788572471.576850:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.576856:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ca1 [1] + 224 00000800:00000001:0.0:1788572471.576863:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.576935:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.576940:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.576945:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.576953:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.576962:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809942144/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.576973:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec70e00 x1875440809942144/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.576999:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.577007:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.577011:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572471.580091:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.580097:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572471.580102:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.580108:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.580115:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1ee0 00000400:00000010:0.0:1788572471.580118:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1ee0. 00000100:00000001:0.0:1788572471.580122:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.580127:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70e00 x1875440809942272/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.580137:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.580140:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440809942272/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.580146:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.580148:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572471.583465:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.583483:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.583488:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.583491:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.583501:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.583510:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.583519:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.583524:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.583528:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.583533:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461d00 00000400:00000200:0.0:1788572471.583538:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.583544:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ca9 [1] + 224 00000800:00000001:0.0:1788572471.583551:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.583559:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.583563:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.583567:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.583575:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.583584:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809942272/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.583595:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70e00 x1875440809942272/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.583615:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.583624:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.583627:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572471.613042:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.613050:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572471.613057:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.613064:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.613070:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f33b8 00000400:00000010:0.0:1788572471.613072:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f33b8. 00000100:00000001:0.0:1788572471.613076:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.613081:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809942400/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.613093:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.613096:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6300 x1875440809942400/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.613110:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.613112:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572471.622206:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.622230:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.622234:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.622238:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.622248:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.622256:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.622268:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.622273:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.622277:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.622281:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417461d80 00000400:00000200:0.0:1788572471.622286:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.622293:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529cb1 [1] + 224 00000800:00000001:0.0:1788572471.622300:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.622312:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.622316:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.622320:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.622328:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.622336:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809942400/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.622348:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6300 x1875440809942400/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.622372:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.622380:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.622383:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.625257:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.625262:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572471.632665:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632685:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.632689:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632692:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.632701:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632709:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.632721:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632727:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632733:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632737:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417461e00 00000400:00000200:0.0:1788572471.632742:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632749:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529cc1 [1] + 224 00000800:00000001:0.0:1788572471.632756:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632766:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.632770:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.632775:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.632783:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.632793:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809942528/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572471.632819:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c6300 x1875440809942528/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.632841:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.632850:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632854:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572471.632856:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632858:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.632861:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632864:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572471.632868:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632876:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632879:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632889:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417461f00 00000400:00000200:0.0:1788572471.632893:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.632898:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x529cd1 [1] + 224 00000800:00000001:0.0:1788572471.632901:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.632951:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572471.632954:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.632957:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572471.632960:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.632965:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6bb80 x1875440809942784/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572556 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572471.632973:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6bb80 x1875440809942784/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572471.632992:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572471.632997:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572471.633001:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.722100:0:261356:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.722108:0:261356:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.722111:0:261356:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.722114:0:261356:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37e18. 00000400:00000200:0.0:1788572471.722120:0:261356:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1e58 00000400:00000010:0.0:1788572471.722122:0:261356:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1e58. 00000100:00000001:0.0:1788572471.722125:0:261356:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.722130:0:261356:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.722138:0:261356:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.722144:0:261356:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.722145:0:261356:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.722148:0:261356:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.722151:0:261356:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.722154:0:261356:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.722156:0:261356:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.722159:0:261356:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.722168:0:261356:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.722173:0:261356:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.722175:0:261356:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.722177:0:261356:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722180:0:261356:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.722184:0:261356:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.722188:0:261356:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.722190:0:261356:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.722193:0:261356:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.722196:0:261356:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.722201:0:261356:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.722208:0:261356:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.722211:0:261356:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.722213:0:261356:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.722217:0:261356:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.722225:0:261356:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261356:1875440809943168:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.722230:0:261356:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.722232:0:261356:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.722235:0:261356:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.722237:0:261356:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.722240:0:261356:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.722247:0:261356:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.722249:0:261356:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c37380b6c00. 00000100:00000001:0.0:1788572471.722253:0:261356:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.722254:0:261356:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722257:0:261356:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.722261:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722264:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722267:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=4 00000020:00000001:0.0:1788572471.722269:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722272:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000040:0.0:1788572471.722279:0:261356:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37380c6a00, remote cookie: 0xd9894672d99a2427, flags: 0x1001 00010000:00010000:0.0:1788572471.722285:0:261356:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.722294:0:261356:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdbe:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.722298:0:261356:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.722305:0:261356:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff400 count: 2 00010000:00000001:0.0:1788572471.722308:0:261356:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.722311:0:261356:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.722314:0:261356:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.722323:0:261356:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722329:0:261356:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.722333:0:261356:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.722335:0:261356:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.722337:0:261356:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.722339:0:261356:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.722342:0:261356:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdbe:0x0].0x0 (ffff9c3713fff400) refcount = 2 00010000:00000040:0.0:1788572471.722344:0:261356:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.722347:0:261356:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c370306c800/0x1441586093167f13 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a109a expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.722354:0:261356:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722361:0:261356:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.722364:0:261356:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.722366:0:261356:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.722369:0:261356:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.722372:0:261356:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.722375:0:261356:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722382:0:261356:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.722383:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722385:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722386:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722387:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722388:0:261356:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.722393:0:261356:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.722394:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722395:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722397:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722399:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722401:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722404:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722405:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722406:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=4 00000020:00000001:0.0:1788572471.722408:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722410:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722412:0:261356:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.722415:0:261356:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.722418:0:261356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722424:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722425:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722426:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722428:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722429:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=5 00000020:00000001:0.0:1788572471.722430:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722432:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722435:0:261356:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.722437:0:261356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722443:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722444:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.722446:0:261356:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722452:0:261356:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.722453:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722454:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722456:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722457:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.722461:0:261356:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.722470:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722471:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722472:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722474:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722476:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00000002:00010000:0.0:1788572471.722480:0:261356:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.722489:0:261356:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091580. 00010000:00000001:0.0:1788572471.722491:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722492:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.722494:0:261356:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.722495:0:261356:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.722498:0:261356:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.722500:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722501:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722502:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722504:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722505:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00000002:00010000:0.0:1788572471.722508:0:261356:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722514:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722516:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722517:0:261356:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.722519:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722520:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722521:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722523:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722525:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000040:0.0:1788572471.722528:0:261356:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff400 count: 3 00010000:00000001:0.0:1788572471.722529:0:261356:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.722533:0:261356:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3518 (0 0) 00010000:00000001:0.0:1788572471.722536:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722537:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.722538:0:261356:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.722541:0:261356:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc208b710 00000002:00002000:0.0:1788572471.722543:0:261356:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-042 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.722545:0:261356:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.722548:0:261356:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.722551:0:261356:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.722555:0:261356:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.722558:0:261356:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.722561:0:261356:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.722564:0:261356:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.722566:0:261356:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc208bb20 it_disposition b 00000080:00000001:0.0:1788572471.722569:0:261356:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.722574:0:261356:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.722577:0:261356:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.722579:0:261356:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329086 : 144115373145329086 : 200002b17000dbe) 00000080:00000001:0.0:1788572471.722581:0:261356:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.722587:0:261356:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.722591:0:261356:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941496 : -109714522610120 : ffff9c3718da6638) 00000020:00001000:0.0:1788572471.722593:0:261356:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6630 00000020:00000001:0.0:1788572471.722596:0:261356:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.722599:0:261356:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.722602:0:261356:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.722607:0:261356:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.722611:0:261356:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.722616:0:261356:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.722618:0:261356:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.722622:0:261356:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdbe:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.722626:0:261356:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.722630:0:261356:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21104 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.722657:0:261356:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf14d8) owner ffff9c37012a0000 00020000:00000001:0.0:1788572471.722660:0:261356:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.722662:0:261356:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf14d8) owner ffff9c37012a0000 00020000:00000010:0.0:1788572471.722666:0:261356:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.722669:0:261356:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.722672:0:261356:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.722674:0:261356:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdbe:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.722676:0:261356:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.722678:0:261356:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.722680:0:261356:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6630 00000080:00000001:0.0:1788572471.722683:0:261356:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329086 : 144115373145329086 : 200002b17000dbe) 00000080:00000001:0.0:1788572471.722686:0:261356:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.722691:0:261356:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdbe:0x0](ffff9c3719f863c8): rc = 0 00000080:00000001:0.0:1788572471.722693:0:261356:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359205684168 : -109714503867448 : ffff9c3719f863c8) 00010000:00000001:0.0:1788572471.722695:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722696:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722697:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722699:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722701:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00000080:00000001:0.0:1788572471.722705:0:261356:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.722706:0:261356:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.722707:0:261356:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941496 : -109714522610120 : ffff9c3718da6638) 00000020:00001000:0.0:1788572471.722709:0:261356:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6630 00000020:00000001:0.0:1788572471.722711:0:261356:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.722712:0:261356:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.722713:0:261356:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.722716:0:261356:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.722718:0:261356:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.722720:0:261356:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.722722:0:261356:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.722724:0:261356:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdbe:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.722728:0:261356:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.722732:0:261356:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21104 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.722734:0:261356:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf14d8) owner ffff9c37012a0000 00020000:00000001:0.0:1788572471.722736:0:261356:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.722738:0:261356:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf14d8) owner ffff9c37012a0000 00020000:00000010:0.0:1788572471.722740:0:261356:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.722741:0:261356:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.722743:0:261356:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.722745:0:261356:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdbe:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.722746:0:261356:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.722748:0:261356:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.722751:0:261356:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.722754:0:261356:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf14d8) owner 0000000000000000/ffff9c37012a0000 00020000:00000002:0.0:1788572471.722756:0:261356:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ec97d80 addref 2/0 by ffff9c37012a0000. 00020000:00000002:0.0:1788572471.722759:0:261356:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf14d8) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572471.722761:0:261356:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.722763:0:261356:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.722764:0:261356:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.722766:0:261356:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.722768:0:261356:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.722770:0:261356:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdbe:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.722772:0:261356:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6630 00000080:00000001:0.0:1788572471.722774:0:261356:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722775:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722776:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.722777:0:261356:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.722780:0:261356:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.722782:0:261356:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.722784:0:261356:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdbe:0x0](ffff9c3719f863c8) for lock 0x1441586093168533 00800000:00000001:0.0:1788572471.722788:0:261356:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.722789:0:261356:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.722791:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722792:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722793:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722794:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722796:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722799:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722800:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.722801:0:261356:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.722803:0:261356:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.722808:0:261356:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c3702109510 inode ffff9c3719f863c8 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.722812:0:261356:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.722828:0:261356:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdbe:0x0](ffff9c3719f863c8) 00000080:00010000:0.0:1788572471.722833:0:261356:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168533 from it ffffb95bc208bb20 00010000:00000001:0.0:1788572471.722835:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.722836:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.722837:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.722839:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722841:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.722843:0:261356:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.722846:0:261356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.722851:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722852:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722854:0:261356:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.722856:0:261356:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.722859:0:261356:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.722871:0:261356:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.722873:0:261356:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.722875:0:261356:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.722879:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722890:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.722892:0:261356:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722895:0:261356:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.722897:0:261356:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722899:0:261356:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722901:0:261356:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.722903:0:261356:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.722904:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.722905:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.722907:0:261356:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.722910:0:261356:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3703114e00. 00000100:00000001:0.0:1788572471.722914:0:261356:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.722918:0:261356:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440809943168/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.722925:0:261356:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.722927:0:261356:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.722931:0:261356:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37051a0000. 02000000:00000001:0.0:1788572471.722933:0:261356:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.722935:0:261356:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.722937:0:261356:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.722939:0:261356:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.722942:0:261356:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788572471.722945:0:261356:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.722947:0:261356:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.722949:0:261356:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788572471.722952:0:261356:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.722953:0:261356:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.722957:0:261356:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.722958:0:261356:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc208bb20 released 00000080:00000001:0.0:1788572471.722960:0:261356:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.722968:0:261356:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc208bd08 00000080:00200008:0.0:1788572471.722971:0:261356:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-042:[0x200002b17:0xdbe:0x0](ffff9c3719f863c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.722977:0:261356:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.722978:0:261356:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc208bc80 00000080:00200000:0.0:1788572471.722980:0:261356:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdbe:0x0](ffff9c3719f863c8),name=f434.sanity-042 00000080:00000010:0.0:1788572471.722985:0:261356:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572471.722989:0:261356:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.722996:0:261356:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.722999:0:261356:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.723000:0:261356:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc208bba0 00800000:00000002:0.0:1788572471.723002:0:261356:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdbe:0x0] 'f434.sanity-042' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.723005:0:261356:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.723009:0:261356:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.723011:0:261356:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.723014:0:261356:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.723016:0:261356:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.723018:0:261356:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.723020:0:261356:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.723022:0:261356:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.723025:0:261356:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.723026:0:261356:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.723027:0:261356:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.723028:0:261356:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.723029:0:261356:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.723031:0:261356:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdbe:0x0] 00800000:00000001:0.0:1788572471.723032:0:261356:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.723034:0:261356:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdbe:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.723038:0:261356:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.723040:0:261356:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc208ba70 00000002:00010000:0.0:1788572471.723041:0:261356:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdbe:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.723044:0:261356:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.723046:0:261356:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.723047:0:261356:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.723050:0:261356:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713fff400 count: 3 00010000:00000001:0.0:1788572471.723052:0:261356:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.723055:0:261356:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.723057:0:261356:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.723060:0:261356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.723067:0:261356:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.723071:0:261356:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.723074:0:261356:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.723080:0:261356:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.723082:0:261356:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.723084:0:261356:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.723085:0:261356:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.723087:0:261356:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.723089:0:261356:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.723091:0:261356:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.723093:0:261356:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.723095:0:261356:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572471.723100:0:261356:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.723102:0:261356:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.723104:0:261356:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdbe:0x0](ffff9c3719f863c8) 00000080:00010000:0.0:1788572471.723107:0:261356:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdbe:0x0](ffff9c3719f863c8) for lock 0x1441586093168533 00800000:00000001:0.0:1788572471.723109:0:261356:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.723111:0:261356:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.723112:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.723113:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.723114:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.723116:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.723118:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.723120:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.723121:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.723122:0:261356:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.723124:0:261356:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.723125:0:261356:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168533 from it ffffb95bc208bcb8 00010000:00000001:0.0:1788572471.723127:0:261356:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.723128:0:261356:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.723129:0:261356:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37380c6a00 refcount=3 00000020:00000001:0.0:1788572471.723131:0:261356:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.723133:0:261356:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359710312960 : -109713999238656 : ffff9c37380c6a00) 00010000:00000001:0.0:1788572471.723134:0:261356:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.723136:0:261356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.723141:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.723142:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.723143:0:261356:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.723144:0:261356:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.723146:0:261356:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37380c6a00/0x1441586093168533 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2427 expref: -99 pid: 261356 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.723151:0:261356:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.723153:0:261356:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.723154:0:261356:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.723156:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.723157:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.723158:0:261356:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723160:0:261356:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.723161:0:261356:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723163:0:261356:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723165:0:261356:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723167:0:261356:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.723168:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.723169:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.723171:0:261356:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.723172:0:261356:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.723175:0:261356:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.723178:0:261356:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.723180:0:261356:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.723182:0:261356:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.723183:0:261356:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.723185:0:261356:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.723188:0:261356:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.723190:0:261356:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.723192:0:261356:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941496 : -109714522610120 : ffff9c3718da6638) 00000020:00001000:0.0:1788572471.723194:0:261356:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6630 00000020:00000001:0.0:1788572471.723199:0:261356:0:(cl_object.c:143:cl_object_top()) ffff9c371f02ae90 -> ffff9c371f02ae90 00000020:00000001:0.0:1788572471.723201:0:261356:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.723204:0:261356:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.723206:0:261356:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdbe:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.723209:0:261356:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.723210:0:261356:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.723212:0:261356:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.723213:0:261356:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.723215:0:261356:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.723219:0:261356:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdbe:0x0]io ffff9c371ff53c80 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.723221:0:261356:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf14d8) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572471.723224:0:261356:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.723226:0:261356:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.723228:0:261356:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.723230:0:261356:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723232:0:261356:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.723234:0:261356:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.723235:0:261356:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723237:0:261356:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.723238:0:261356:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.723239:0:261356:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.723241:0:261356:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf14d8) owner 0000000000000000/ffff9c37012a0000 00000020:00000001:0.0:1788572471.723243:0:261356:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.723245:0:261356:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.723246:0:261356:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdbe:0x0] 00000020:00000001:0.0:1788572471.723249:0:261356:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.723251:0:261356:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.723253:0:261356:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.723255:0:261356:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.723260:0:261356:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.723265:0:261356:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676c00. 00020000:00000002:0.0:1788572471.723267:0:261356:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.723269:0:261356:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.723272:0:261356:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.723273:0:261356:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.723274:0:261356:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.723276:0:261356:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943512 : -109714522608104 : ffff9c3718da6e18) 00000020:00001000:0.0:1788572471.723277:0:261356:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6e10 00000020:00000001:0.0:1788572471.723280:0:261356:0:(cl_object.c:143:cl_object_top()) ffff9c371f02a788 -> ffff9c371f02ae90 00000020:00000001:0.0:1788572471.723281:0:261356:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.723284:0:261356:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.723285:0:261356:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.723286:0:261356:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723288:0:261356:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723290:0:261356:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293057992 : -109714416493624 : ffff9c371f2d9bc8) 00000020:00000001:0.0:1788572471.723292:0:261356:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.723296:0:261356:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a70a0. 00000008:00010000:0.0:1788572471.723298:0:261356:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.723300:0:261356:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.723302:0:261356:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.723303:0:261356:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676c30, osc lock ffff9c37030a70a0, flags 0x400001002 00000020:00000001:0.0:1788572471.723305:0:261356:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723307:0:261356:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723309:0:261356:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531727872 : -109714177823744 : ffff9c372d676c00) 00000020:00000001:0.0:1788572471.723311:0:261356:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.723312:0:261356:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.723313:0:261356:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723315:0:261356:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723317:0:261356:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.723319:0:261356:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.723320:0:261356:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.723322:0:261356:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.723324:0:261356:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.723325:0:261356:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293057992 : -109714416493624 : ffff9c371f2d9bc8) 00000020:00000001:0.0:1788572471.723327:0:261356:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.723329:0:261356:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.723331:0:261356:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.723334:0:261356:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.723336:0:261356:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.723337:0:261356:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.723339:0:261356:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.723340:0:261356:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.723341:0:261356:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.723344:0:261356:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.723346:0:261356:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.723350:0:261356:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723352:0:261356:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.723356:0:261356:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.723360:0:261356:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.723363:0:261356:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.723368:0:261356:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.723372:0:261356:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.723373:0:261356:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.723374:0:261356:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.723377:0:261356:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.723379:0:261356:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.723382:0:261356:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.723387:0:261356:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572471.723394:0:261356:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723398:0:261356:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.723399:0:261356:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723401:0:261356:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.723402:0:261356:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723404:0:261356:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.723406:0:261356:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.723409:0:261356:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.723411:0:261356:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.723417:0:261356:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0600. 00010000:00000010:0.0:1788572471.723420:0:261356:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0200. 00010000:00000001:0.0:1788572471.723424:0:261356:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.723426:0:261356:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b000. 00000020:00000001:0.0:1788572471.723430:0:261356:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.723432:0:261356:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b000 with handle 0x144158609316853a to hash 00000020:00000001:0.0:1788572471.723434:0:261356:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.723436:0:261356:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000010:0.0:1788572471.723440:0:261356:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091b00. 00010000:00000001:0.0:1788572471.723442:0:261356:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.723445:0:261356:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.723447:0:261356:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.723450:0:261356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.723459:0:261356:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.723469:0:261356:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.723473:0:261356:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.723477:0:261356:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.723484:0:261356:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.723524:0:261356:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.723527:0:261356:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723529:0:261356:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.723531:0:261356:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723532:0:261356:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.723535:0:261356:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.723536:0:261356:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.723538:0:261356:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.723540:0:261356:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.723553:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572471.723556:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572471.723558:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.723565:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.723567:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.723569:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.723570:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.723573:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.723579:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.723587:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.723589:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.723594:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.723597:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.723599:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.723602:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809943296:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.723606:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.723612:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.723614:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.723617:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.723622:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788572471.723624:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.723633:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.723661:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.723668:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.723675:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809943296, portal 4 00000100:00000001:0.0:1788572471.723678:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.723679:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.723685:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.723693:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.723695:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.723698:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572471.723701:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809943296, offset 0 00000400:00000200:0.0:1788572471.723708:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.723716:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.723723:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.723728:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.723732:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.723735:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.723738:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.723741:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539749:539749:256:4294967295] 192.168.202.148@tcp LPNI seq info [539714:539714:8:4294967295] 00000400:00000200:0.0:1788572471.723746:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.723755:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.723761:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.723768:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572471.723774:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.723782:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.723786:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.723796:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.723800:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 169726 usecs (170 jiffies) 00000400:00000200:0.0:1788572471.723804:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.723806:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610898 00000100:00000001:0.0:1788572471.723809:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.723811:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610898 00000100:00000001:0.0:1788572471.723813:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.723816:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.723818:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.723821:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.723823:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.734789:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.734794:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.734802:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.734804:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.734808:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.734813:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.734932:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.734939:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.734946:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.734956:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.734963:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.734965:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.734968:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.734971:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.734973:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.734976:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.734979:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.734988:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.734992:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=86 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572471.734994:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.734996:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.734999:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.735003:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.735006:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.735008:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572471.735010:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.735013:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.735017:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.735024:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.735027:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.735029:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.735031:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.735034:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735037:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735040:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=4 00000020:00000001:0.0:1788572471.735042:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735045:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735048:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735049:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735050:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=5 00000020:00000001:0.0:1788572471.735052:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735054:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735057:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.735061:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.735065:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735083:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735084:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.735086:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.735088:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735090:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735091:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=6 00000020:00000001:0.0:1788572471.735093:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735095:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000040:0.0:1788572471.735102:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6b000, remote cookie: 0xd9894672d99a242e, flags: 0x1001 00010000:00010000:0.0:1788572471.735107:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735114:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.735117:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735119:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.735122:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.735126:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.735134:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x5270:0x0].0x0 (ffff9c37198c0600) refcount = 1 00010000:00000040:0.0:1788572471.735137:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.735140:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735149:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.735151:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.735154:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.735156:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.735159:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735167:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.735168:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735169:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.735170:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735171:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.735173:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.735176:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.735179:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735180:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735182:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=5 00000020:00000001:0.0:1788572471.735183:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735185:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00000008:00000001:0.0:1788572471.735190:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.735195:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac540 -> ffff9c371f02ae90 00000008:00010000:0.0:1788572471.735200:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.735207:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac540 -> ffff9c371f02ae90 00000020:00000001:0.0:1788572471.735208:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.735212:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.735215:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.735218:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.735220:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.735222:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac540 -> ffff9c371f02ae90 00000008:00000001:0.0:1788572471.735223:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.735227:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.735229:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.735247:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.735249:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735251:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735252:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735254:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=6 00000020:00000001:0.0:1788572471.735255:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735258:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735261:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.735264:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735272:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735273:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.735276:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735290:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.735291:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735292:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.735294:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735299:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.735301:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.735305:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b000 refcount=5 00000020:00000001:0.0:1788572471.735311:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735314:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699840 : -109714155851776 : ffff9c372eb6b000) 00010000:00000001:0.0:1788572471.735317:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.735321:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.735334:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735337:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.735341:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.735347:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.735352:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005cd2423e/0x144158609316853a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x5270:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a242e expref: -99 pid: 261356 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.735370:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.735375:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.735380:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.735387:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735391:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.735395:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735399:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.735403:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735407:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735412:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.735416:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.735419:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735423:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.735426:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.735429:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.735433:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.735444:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.735459:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809943296:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.735470:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.735475:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.735482:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809943296/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.735490:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.735494:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.735500:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788572471.735506:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.735510:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.735514:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.735520:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.735524:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7000. 02000000:00000001:0.0:1788572471.735532:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.735536:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.735541:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.735547:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.735550:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.735555:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.735566:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.735570:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.735574:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.735578:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.834345:0:261362:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.834407:0:261362:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.834411:0:261362:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.834415:0:261362:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2160. 00000400:00000200:0.0:1788572471.834420:0:261362:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3d48 00000400:00000010:0.0:1788572471.834423:0:261362:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3d48. 00000100:00000001:0.0:1788572471.834427:0:261362:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.834432:0:261362:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.834440:0:261362:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.834446:0:261362:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.834447:0:261362:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.834450:0:261362:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.834453:0:261362:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.834455:0:261362:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.834458:0:261362:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.834461:0:261362:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.834471:0:261362:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.834475:0:261362:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.834478:0:261362:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.834479:0:261362:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.834482:0:261362:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.834486:0:261362:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.834489:0:261362:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.834491:0:261362:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.834495:0:261362:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.834497:0:261362:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.834502:0:261362:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.834515:0:261362:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.834517:0:261362:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.834519:0:261362:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.834524:0:261362:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.834531:0:261362:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261362:1875440809943936:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.834536:0:261362:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.834539:0:261362:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.834541:0:261362:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.834543:0:261362:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.834547:0:261362:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.834565:0:261362:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.834568:0:261362:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c372d946840. 00000100:00000001:0.0:1788572471.834571:0:261362:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.834573:0:261362:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.834576:0:261362:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.834579:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.834582:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.834585:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=4 00000020:00000001:0.0:1788572471.834588:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834591:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000040:0.0:1788572471.834598:0:261362:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372da6ea00, remote cookie: 0xd9894672d99a2490, flags: 0x1001 00010000:00010000:0.0:1788572471.834603:0:261362:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.834611:0:261362:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdc1:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.834614:0:261362:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.834620:0:261362:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7b00 count: 2 00010000:00000001:0.0:1788572471.834624:0:261362:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.834628:0:261362:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.834632:0:261362:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.834751:0:261362:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834757:0:261362:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.834761:0:261362:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.834763:0:261362:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.834765:0:261362:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.834767:0:261362:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.834769:0:261362:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdc1:0x0].0x0 (ffff9c37050f7b00) refcount = 2 00010000:00000040:0.0:1788572471.834772:0:261362:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.834774:0:261362:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cbc00/0x1441586093167f3d lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a112d expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.834781:0:261362:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834787:0:261362:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.834791:0:261362:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.834793:0:261362:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.834795:0:261362:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.834798:0:261362:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.834802:0:261362:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834807:0:261362:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.834809:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834811:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.834812:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834814:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.834815:0:261362:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.834896:0:261362:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.834898:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.834899:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.834901:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.834903:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834906:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834908:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.834909:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.834910:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=4 00000020:00000001:0.0:1788572471.834912:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834913:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834916:0:261362:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.834919:0:261362:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.834922:0:261362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834928:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834930:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.834931:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.834932:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.834933:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=5 00000020:00000001:0.0:1788572471.834935:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834937:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834939:0:261362:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.834942:0:261362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834948:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834949:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.834951:0:261362:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.834962:0:261362:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.834963:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834964:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.834965:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.834966:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.834971:0:261362:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.834981:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.834982:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.834984:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.834986:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.834988:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00000002:00010000:0.0:1788572471.834992:0:261362:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.835001:0:261362:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091000. 00010000:00000001:0.0:1788572471.835003:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.835004:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.835006:0:261362:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.835008:0:261362:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.835011:0:261362:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.835013:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.835014:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.835016:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.835017:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.835019:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00000002:00010000:0.0:1788572471.835022:0:261362:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.835028:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.835030:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.835032:0:261362:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.835033:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.835034:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.835035:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.835038:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.835039:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000040:0.0:1788572471.835042:0:261362:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7b00 count: 3 00010000:00000001:0.0:1788572471.835044:0:261362:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.835049:0:261362:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3521 (0 0) 00010000:00000001:0.0:1788572471.835051:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.835052:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.835053:0:261362:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.835056:0:261362:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9f710 00000002:00002000:0.0:1788572471.835058:0:261362:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-045 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.835061:0:261362:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.835064:0:261362:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.835067:0:261362:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.835070:0:261362:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.835138:0:261362:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.835146:0:261362:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.835149:0:261362:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.835151:0:261362:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1e9fb20 it_disposition b 00000080:00000001:0.0:1788572471.835155:0:261362:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.835162:0:261362:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.835165:0:261362:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.835167:0:261362:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329089 : 144115373145329089 : 200002b17000dc1) 00000080:00000001:0.0:1788572471.835171:0:261362:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.835180:0:261362:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.835185:0:261362:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572471.835187:0:261362:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572471.835190:0:261362:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.835193:0:261362:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.835196:0:261362:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.835201:0:261362:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.835205:0:261362:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.835212:0:261362:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.835214:0:261362:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.835217:0:261362:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdc1:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.835223:0:261362:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.835227:0:261362:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20991 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.835230:0:261362:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf17c0) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572471.835234:0:261362:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.835236:0:261362:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf17c0) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572471.835239:0:261362:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.835243:0:261362:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.835246:0:261362:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.838242:0:261362:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc1:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.838246:0:261362:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.838250:0:261362:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.838254:0:261362:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572471.838262:0:261362:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329089 : 144115373145329089 : 200002b17000dc1) 00000080:00000001:0.0:1788572471.838265:0:261362:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.838270:0:261362:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdc1:0x0](ffff9c3719f84b08): rc = 0 00000080:00000001:0.0:1788572471.838273:0:261362:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634359205677832 : -109714503873784 : ffff9c3719f84b08) 00010000:00000001:0.0:1788572471.838277:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.838280:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.838283:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.838287:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838290:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00000080:00000001:0.0:1788572471.838294:0:261362:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.838296:0:261362:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.838298:0:261362:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572471.838301:0:261362:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572471.838302:0:261362:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.838304:0:261362:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.838306:0:261362:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.838312:0:261362:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.838318:0:261362:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.838324:0:261362:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.838326:0:261362:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.838329:0:261362:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdc1:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.838334:0:261362:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.838338:0:261362:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20991 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.838342:0:261362:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3703cf17c0) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572471.838345:0:261362:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.838347:0:261362:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3703cf17c0) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572471.838351:0:261362:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.838355:0:261362:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.838357:0:261362:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.838359:0:261362:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc1:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.838361:0:261362:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.838362:0:261362:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.838366:0:261362:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.838370:0:261362:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf17c0) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000002:0.0:1788572471.838373:0:261362:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ec97cc0 addref 2/0 by ffff9c372d1c4f80. 00020000:00000002:0.0:1788572471.838376:0:261362:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf17c0) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572471.838378:0:261362:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.838380:0:261362:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.838382:0:261362:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.838384:0:261362:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.838386:0:261362:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.838389:0:261362:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdc1:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.838391:0:261362:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da61b0 00000080:00000001:0.0:1788572471.838394:0:261362:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838396:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838398:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.838400:0:261362:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.838405:0:261362:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.838408:0:261362:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.838412:0:261362:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc1:0x0](ffff9c3719f84b08) for lock 0x144158609316855d 00800000:00000001:0.0:1788572471.838416:0:261362:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.838420:0:261362:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.838421:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.838422:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.838424:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.838425:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838427:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838430:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838431:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.838433:0:261362:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.838435:0:261362:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.838441:0:261362:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37061cc288 inode ffff9c3719f84b08 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.838445:0:261362:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.838449:0:261362:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc1:0x0](ffff9c3719f84b08) 00000080:00010000:0.0:1788572471.838453:0:261362:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316855d from it ffffb95bc1e9fb20 00010000:00000001:0.0:1788572471.838455:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.838456:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.838458:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.838459:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838461:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838464:0:261362:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.838468:0:261362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.838477:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838478:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.838481:0:261362:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.838484:0:261362:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.838486:0:261362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.838494:0:261362:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.838497:0:261362:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.838500:0:261362:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.838505:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838506:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.838508:0:261362:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838511:0:261362:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.838513:0:261362:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838515:0:261362:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838520:0:261362:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838522:0:261362:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.838523:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838524:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.838526:0:261362:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.838530:0:261362:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711215e00. 00000100:00000001:0.0:1788572471.838537:0:261362:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.838550:0:261362:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440809943936/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.838558:0:261362:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.838561:0:261362:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.838564:0:261362:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c3705364800. 02000000:00000001:0.0:1788572471.838568:0:261362:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.838571:0:261362:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.838572:0:261362:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.838574:0:261362:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.838577:0:261362:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371fe97800. 02000000:00000001:0.0:1788572471.838581:0:261362:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.838583:0:261362:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.838584:0:261362:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788572471.838588:0:261362:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.838589:0:261362:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.838592:0:261362:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.838593:0:261362:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc1e9fb20 released 00000080:00000001:0.0:1788572471.838595:0:261362:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.838605:0:261362:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fd08 00000080:00200008:0.0:1788572471.838607:0:261362:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-045:[0x200002b17:0xdc1:0x0](ffff9c3719f84b08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.838613:0:261362:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.838614:0:261362:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fc80 00000080:00200000:0.0:1788572471.838616:0:261362:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdc1:0x0](ffff9c3719f84b08),name=f434.sanity-045 00000080:00000010:0.0:1788572471.838622:0:261362:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572471.838628:0:261362:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.838665:0:261362:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.838670:0:261362:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.838672:0:261362:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fba0 00800000:00000002:0.0:1788572471.838674:0:261362:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdc1:0x0] 'f434.sanity-045' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.838679:0:261362:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.838683:0:261362:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.838686:0:261362:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.838689:0:261362:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.838692:0:261362:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.838694:0:261362:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.838696:0:261362:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.838698:0:261362:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.838700:0:261362:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.838701:0:261362:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.838702:0:261362:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.838703:0:261362:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.838705:0:261362:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.838707:0:261362:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdc1:0x0] 00800000:00000001:0.0:1788572471.838708:0:261362:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.838711:0:261362:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdc1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.838715:0:261362:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.838716:0:261362:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fa70 00000002:00010000:0.0:1788572471.838718:0:261362:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdc1:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.838722:0:261362:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.838723:0:261362:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.838725:0:261362:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.838732:0:261362:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7b00 count: 3 00010000:00000001:0.0:1788572471.838734:0:261362:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.838738:0:261362:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.838741:0:261362:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.838745:0:261362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.838757:0:261362:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.838761:0:261362:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.838764:0:261362:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.838771:0:261362:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.838773:0:261362:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.838775:0:261362:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.838776:0:261362:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.838778:0:261362:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.838780:0:261362:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.838783:0:261362:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.838785:0:261362:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.838787:0:261362:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572471.838793:0:261362:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.838794:0:261362:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.838796:0:261362:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc1:0x0](ffff9c3719f84b08) 00000080:00010000:0.0:1788572471.838799:0:261362:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc1:0x0](ffff9c3719f84b08) for lock 0x144158609316855d 00800000:00000001:0.0:1788572471.838801:0:261362:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.838803:0:261362:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.838804:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.838805:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.838807:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.838808:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838810:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838812:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838813:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.838814:0:261362:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.838815:0:261362:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.838898:0:261362:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316855d from it ffffb95bc1e9fcb8 00010000:00000001:0.0:1788572471.838901:0:261362:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.838903:0:261362:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.838904:0:261362:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ea00 refcount=3 00000020:00000001:0.0:1788572471.838907:0:261362:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838909:0:261362:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535888896 : -109714173662720 : ffff9c372da6ea00) 00010000:00000001:0.0:1788572471.838911:0:261362:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.838914:0:261362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.838921:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838922:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.838923:0:261362:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.838925:0:261362:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.838927:0:261362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ea00/0x144158609316855d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2490 expref: -99 pid: 261362 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.838932:0:261362:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.838934:0:261362:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.838936:0:261362:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.838938:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838939:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.838940:0:261362:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838942:0:261362:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.838944:0:261362:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838945:0:261362:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838948:0:261362:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.838949:0:261362:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.838950:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.838951:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.838953:0:261362:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.838955:0:261362:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.838958:0:261362:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.838960:0:261362:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.838962:0:261362:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.838965:0:261362:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.838966:0:261362:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.838969:0:261362:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.838972:0:261362:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.838974:0:261362:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.838976:0:261362:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940344 : -109714522611272 : ffff9c3718da61b8) 00000020:00001000:0.0:1788572471.838978:0:261362:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da61b0 00000020:00000001:0.0:1788572471.838982:0:261362:0:(cl_object.c:143:cl_object_top()) ffff9c371f02a5f8 -> ffff9c371f02a5f8 00000020:00000001:0.0:1788572471.838985:0:261362:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.838987:0:261362:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.838990:0:261362:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdc1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.838993:0:261362:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.838994:0:261362:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.838996:0:261362:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.838997:0:261362:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.838999:0:261362:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.839003:0:261362:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdc1:0x0]io ffff9c371ff53590 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.839006:0:261362:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3703cf17c0) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572471.839010:0:261362:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.839012:0:261362:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.839014:0:261362:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.839016:0:261362:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839018:0:261362:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.839020:0:261362:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.839022:0:261362:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839023:0:261362:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.839024:0:261362:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.839025:0:261362:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.839026:0:261362:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3703cf17c0) owner 0000000000000000/ffff9c372d1c4f80 00000020:00000001:0.0:1788572471.839028:0:261362:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.839030:0:261362:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.839032:0:261362:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdc1:0x0] 00000020:00000001:0.0:1788572471.839034:0:261362:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.839036:0:261362:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.839038:0:261362:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.839040:0:261362:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.839045:0:261362:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.839050:0:261362:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676280. 00020000:00000002:0.0:1788572471.839053:0:261362:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.839055:0:261362:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.839057:0:261362:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.839058:0:261362:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.839060:0:261362:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.839062:0:261362:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359600765624 : -109714108785992 : ffff9c373184dab8) 00000020:00001000:0.0:1788572471.839063:0:261362:0:(cl_object.c:823:cl_env_get()) 1@ffff9c373184dab0 00000020:00000001:0.0:1788572471.839066:0:261362:0:(cl_object.c:143:cl_object_top()) ffff9c371f02a2d8 -> ffff9c371f02a5f8 00000020:00000001:0.0:1788572471.839068:0:261362:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.839071:0:261362:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.839072:0:261362:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.839073:0:261362:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839253:0:261362:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839256:0:261362:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293051568 : -109714416500048 : ffff9c371f2d82b0) 00000020:00000001:0.0:1788572471.839259:0:261362:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.839265:0:261362:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a64c0. 00000008:00010000:0.0:1788572471.839268:0:261362:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.839270:0:261362:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.839271:0:261362:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.839273:0:261362:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d6762b0, osc lock ffff9c37030a64c0, flags 0x400001002 00000020:00000001:0.0:1788572471.839275:0:261362:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839277:0:261362:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839280:0:261362:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725440 : -109714177826176 : ffff9c372d676280) 00000020:00000001:0.0:1788572471.839282:0:261362:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.839284:0:261362:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.839285:0:261362:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839287:0:261362:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839290:0:261362:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.839292:0:261362:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.839293:0:261362:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.839295:0:261362:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.839296:0:261362:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.839297:0:261362:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293051568 : -109714416500048 : ffff9c371f2d82b0) 00000020:00000001:0.0:1788572471.839299:0:261362:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.839301:0:261362:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.839303:0:261362:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.839308:0:261362:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.839310:0:261362:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.839312:0:261362:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.839314:0:261362:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.839315:0:261362:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.839316:0:261362:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.839319:0:261362:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.839321:0:261362:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.839325:0:261362:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.839328:0:261362:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.839332:0:261362:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.839337:0:261362:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.839341:0:261362:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.839345:0:261362:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.839349:0:261362:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.839350:0:261362:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.839351:0:261362:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.839354:0:261362:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.839356:0:261362:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.839359:0:261362:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.839365:0:261362:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7200. 00000100:00000001:0.0:1788572471.839372:0:261362:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.839377:0:261362:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.839379:0:261362:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.839381:0:261362:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.839382:0:261362:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.839384:0:261362:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.839387:0:261362:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.839390:0:261362:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.839392:0:261362:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.839396:0:261362:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0000. 00010000:00000010:0.0:1788572471.839400:0:261362:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0b00. 00010000:00000001:0.0:1788572471.839403:0:261362:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.839406:0:261362:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6be00. 00000020:00000001:0.0:1788572471.839410:0:261362:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.839413:0:261362:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6be00 with handle 0x1441586093168564 to hash 00000020:00000001:0.0:1788572471.839414:0:261362:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.839416:0:261362:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000010:0.0:1788572471.839420:0:261362:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091900. 00010000:00000001:0.0:1788572471.839422:0:261362:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.839425:0:261362:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.839427:0:261362:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.839430:0:261362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.839440:0:261362:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.839452:0:261362:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.839456:0:261362:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.839460:0:261362:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.839469:0:261362:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.839501:0:261362:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.839504:0:261362:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839506:0:261362:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.839508:0:261362:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839509:0:261362:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.839511:0:261362:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.839513:0:261362:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.839515:0:261362:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.839517:0:261362:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.839527:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.839530:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.839532:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.839537:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.839545:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.839548:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.839552:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.839555:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.839557:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.839560:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809944064:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.839565:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.839571:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.839573:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.839576:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.839580:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.839582:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.839590:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.839594:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.839601:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.839609:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809944064, portal 4 00000100:00000001:0.0:1788572471.839612:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.839613:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.839620:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.839626:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.839629:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.839632:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572471.839708:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809944064, offset 0 00000400:00000200:0.0:1788572471.839717:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.839726:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.839733:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.839740:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.839745:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.839749:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.839752:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.839756:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539755:539755:256:4294967295] 192.168.202.148@tcp LPNI seq info [539720:539720:8:4294967295] 00000400:00000200:0.0:1788572471.839761:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.839770:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.839777:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.839783:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788572471.839790:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.839797:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.839802:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.839813:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.839896:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 179417 usecs (180 jiffies) 00000400:00000200:0.0:1788572471.839901:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.839903:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307611026 00000100:00000001:0.0:1788572471.839907:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.839909:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307611026 00000100:00000001:0.0:1788572471.839911:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.839914:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.839916:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.839921:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.839923:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.839929:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.839931:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572471.839933:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.839935:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.839936:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.839939:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.846867:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.846872:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.846879:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.846893:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.846897:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.846903:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.846905:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.846908:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.846914:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.846922:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.846929:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.846931:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.846933:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.846936:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.846939:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.846942:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.846945:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.846954:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.846959:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=8 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572471.846961:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.846963:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.846966:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.846970:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.846973:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.846976:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572471.846978:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.846980:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.846985:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.846992:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.846995:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.846997:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.847000:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.847003:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847006:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847009:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=4 00000020:00000001:0.0:1788572471.847012:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847016:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847018:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847019:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847021:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=5 00000020:00000001:0.0:1788572471.847023:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847025:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847028:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.847032:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.847037:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847055:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847057:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.847059:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.847061:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847062:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847063:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=6 00000020:00000001:0.0:1788572471.847065:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847067:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000040:0.0:1788572471.847087:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6be00, remote cookie: 0xd9894672d99a2497, flags: 0x1001 00010000:00010000:0.0:1788572471.847094:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847102:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.847105:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.847108:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.847111:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.847116:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.847125:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x51ff:0x0].0x0 (ffff9c37198c0000) refcount = 1 00010000:00000040:0.0:1788572471.847127:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.847130:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847139:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.847140:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.847143:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.847146:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.847150:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847158:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.847160:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847161:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.847162:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847163:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.847165:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.847168:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.847171:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847173:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847175:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=5 00000020:00000001:0.0:1788572471.847177:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847179:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00000008:00000001:0.0:1788572471.847184:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.847188:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad880 -> ffff9c371f02a5f8 00000008:00010000:0.0:1788572471.847194:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.847202:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad880 -> ffff9c371f02a5f8 00000020:00000001:0.0:1788572471.847204:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.847207:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.847210:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.847213:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.847215:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.847217:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ad880 -> ffff9c371f02a5f8 00000008:00000001:0.0:1788572471.847218:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.847221:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.847223:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.847242:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.847245:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.847247:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847248:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847249:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=6 00000020:00000001:0.0:1788572471.847251:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847254:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847256:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.847259:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847266:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847267:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.847270:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847277:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.847278:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847279:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.847280:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.847282:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.847283:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.847284:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6be00 refcount=5 00000020:00000001:0.0:1788572471.847286:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847288:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553703424 : -109714155848192 : ffff9c372eb6be00) 00010000:00000001:0.0:1788572471.847290:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.847291:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.847298:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.847299:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.847300:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.847977:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.847984:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cc3c3444/0x1441586093168564 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a2497 expref: -99 pid: 261362 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.847995:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.847999:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.848002:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.848006:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.848008:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.848009:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.848013:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.848015:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.848017:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.848019:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.848021:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.848023:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.848024:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.848025:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.848026:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.848028:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.848037:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.848046:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809944064:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.848052:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.848055:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.848059:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809944064/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.848065:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.848068:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.848071:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.848075:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.848078:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.848080:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.848082:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.848085:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572471.848090:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.848092:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.848093:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.848097:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.848098:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.848101:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.848108:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.848110:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.848113:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.848115:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.861464:0:261364:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.861472:0:261364:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.861474:0:261364:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.861477:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb20 00000080:00200000:0.0:1788572471.861483:0:261364:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.861492:0:261364:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572471.861499:0:261364:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.861503:0:261364:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.861509:0:261364:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.861511:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba40 00800000:00000002:0.0:1788572471.861513:0:261364:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.861517:0:261364:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.861520:0:261364:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.861523:0:261364:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.861525:0:261364:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.861527:0:261364:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.861529:0:261364:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.861531:0:261364:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.861533:0:261364:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.861537:0:261364:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.861542:0:261364:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.861543:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b910 00000002:00010000:0.0:1788572471.861545:0:261364:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.861549:0:261364:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.861551:0:261364:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.861554:0:261364:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.861561:0:261364:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.861564:0:261364:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.861569:0:261364:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.861573:0:261364:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.861579:0:261364:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 190 lru_type: 1 02000000:00000001:0.0:1788572471.861590:0:261364:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.861596:0:261364:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.861599:0:261364:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 190 lru_type: 1 00000002:00000001:0.0:1788572471.861607:0:261364:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.861609:0:261364:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.861611:0:261364:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.861613:0:261364:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.861615:0:261364:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.861617:0:261364:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.861620:0:261364:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.861623:0:261364:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.861626:0:261364:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572471.861632:0:261364:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.861655:0:261364:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.861658:0:261364:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.861662:0:261364:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.861666:0:261364:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.861668:0:261364:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.861670:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.861673:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.861676:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.861678:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.861681:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.861684:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.861685:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.861687:0:261364:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.861688:0:261364:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.861691:0:261364:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bb58 00010000:00000001:0.0:1788572471.861695:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.861696:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.861699:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.861701:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.861703:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.861706:0:261364:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.861710:0:261364:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 190 lru_type: 1 00010000:00000001:0.0:1788572471.861716:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.861717:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.861718:0:261364:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.861720:0:261364:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.861723:0:261364:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 191 lru_type: 1 00010000:00000001:0.0:1788572471.861730:0:261364:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.861734:0:261364:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.861738:0:261364:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.861744:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.861745:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.861746:0:261364:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861750:0:261364:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.861752:0:261364:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861754:0:261364:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861757:0:261364:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861759:0:261364:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.861760:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.861761:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.861765:0:261364:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.861766:0:261364:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.861769:0:261364:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.861773:0:261364:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.861777:0:261364:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.861779:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00002000:0.0:1788572471.861781:0:261364:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 45 00000080:00000001:0.0:1788572471.861785:0:261364:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.861787:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb30 00000080:00200000:0.0:1788572471.861788:0:261364:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572471.861794:0:261364:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572471.861798:0:261364:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572471.861799:0:261364:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572471.861802:0:261364:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572471.861810:0:261364:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f8e0. 00000020:00000001:0.0:1788572471.861814:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00200000:0.0:1788572471.861815:0:261364:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-046, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572471.861819:0:261364:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572471.861820:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b998 00000080:00200000:0.0:1788572471.861822:0:261364:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-046, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572471.861829:0:261364:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572471.861831:0:261364:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.861836:0:261364:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.861837:0:261364:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.861839:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b8b8 00800000:00000002:0.0:1788572471.861840:0:261364:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-046' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.861843:0:261364:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.861846:0:261364:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.861847:0:261364:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.861848:0:261364:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.861850:0:261364:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.861851:0:261364:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.861853:0:261364:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.861855:0:261364:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.861857:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b850 00800000:00000002:0.0:1788572471.861859:0:261364:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-046' -> mds #0 00000002:00000001:0.0:1788572471.861862:0:261364:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.861863:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b788 00000002:00010000:0.0:1788572471.861864:0:261364:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-046,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572471.861868:0:261364:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572471.861870:0:261364:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572471.861874:0:261364:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.861878:0:261364:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572471.861897:0:261364:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.861901:0:261364:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.861903:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.861904:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.861906:0:261364:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.861908:0:261364:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.861910:0:261364:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.861914:0:261364:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.861921:0:261364:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572471.861929:0:261364:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861935:0:261364:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.861937:0:261364:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861939:0:261364:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.861940:0:261364:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.861942:0:261364:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.861949:0:261364:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.861952:0:261364:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.861955:0:261364:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572471.861958:0:261364:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572471.861960:0:261364:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.861963:0:261364:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.861967:0:261364:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b200. 00000020:00000001:0.0:1788572471.861970:0:261364:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.861973:0:261364:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b200 with handle 0x144158609316856b to hash 00000020:00000001:0.0:1788572471.861976:0:261364:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.861977:0:261364:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.861982:0:261364:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.861984:0:261364:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.861987:0:261364:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.861990:0:261364:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261364 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.861999:0:261364:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261364 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.862013:0:261364:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261364 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572471.862018:0:261364:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572471.862020:0:261364:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572471.862026:0:261364:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.862028:0:261364:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572471.862030:0:261364:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.862032:0:261364:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572471.862036:0:261364:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.862040:0:261364:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.862042:0:261364:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572471.862043:0:261364:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.862053:0:261364:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.862062:0:261364:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.862064:0:261364:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.862068:0:261364:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.862070:0:261364:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.862072:0:261364:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.862076:0:261364:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261364:1875440809944192:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.862080:0:261364:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.862084:0:261364:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.862087:0:261364:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.862088:0:261364:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.862093:0:261364:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.862094:0:261364:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.862102:0:261364:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.862105:0:261364:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.862112:0:261364:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.862118:0:261364:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809944192, portal 10 00000100:00000001:0.0:1788572471.862120:0:261364:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.862121:0:261364:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.862126:0:261364:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.862133:0:261364:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.862136:0:261364:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.862138:0:261364:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572471.862140:0:261364:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809944192, offset 0 00000400:00000200:0.0:1788572471.862146:0:261364:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.862154:0:261364:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.862161:0:261364:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.862165:0:261364:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.862169:0:261364:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.862173:0:261364:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.862176:0:261364:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.862180:0:261364:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539756:539756:256:4294967295] 192.168.202.148@tcp LPNI seq info [539721:539721:8:4294967295] 00000400:00000200:0.0:1788572471.862185:0:261364:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.862194:0:261364:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.862200:0:261364:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.862204:0:261364:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788572471.862211:0:261364:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.862219:0:261364:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.862223:0:261364:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0b00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.862246:0:261364:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.862251:0:261364:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14988 usecs (15 jiffies) 00000400:00000200:0.0:1788572471.862254:0:261364:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.862256:0:261364:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610883 00000100:00000001:0.0:1788572471.862259:0:261364:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.862261:0:261364:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610883 00000100:00000001:0.0:1788572471.862264:0:261364:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.862268:0:261364:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.862270:0:261364:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.862272:0:261364:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.862274:0:261364:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572471.862277:0:261364:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572471.862279:0:261364:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.862282:0:261364:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.868492:0:261364:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.870385:0:261364:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.870391:0:261364:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.870394:0:261364:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.870402:0:261364:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.870405:0:261364:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.870409:0:261364:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.870419:0:261364:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.870431:0:261364:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.870438:0:261364:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.870441:0:261364:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.870444:0:261364:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.870447:0:261364:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.870450:0:261364:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.870452:0:261364:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.870455:0:261364:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.870464:0:261364:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.870468:0:261364:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.870471:0:261364:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.870473:0:261364:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.870475:0:261364:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.870480:0:261364:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.870483:0:261364:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.870486:0:261364:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.870488:0:261364:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.870492:0:261364:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.870497:0:261364:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.870504:0:261364:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.870507:0:261364:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.870510:0:261364:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.870513:0:261364:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.870521:0:261364:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261364:1875440809944192:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.870527:0:261364:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.870529:0:261364:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.870531:0:261364:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.870534:0:261364:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.870537:0:261364:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.870544:0:261364:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.870547:0:261364:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.870550:0:261364:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.870552:0:261364:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.870555:0:261364:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.870558:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870561:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870565:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=4 00000020:00000001:0.0:1788572471.870567:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870570:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000040:0.0:1788572471.870577:0:261364:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6b200, remote cookie: 0xd9894672d99a24b3, flags: 0x1001 00010000:00010000:0.0:1788572471.870582:0:261364:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.870591:0:261364:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdc2:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.870595:0:261364:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.870603:0:261364:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514d00 count: 2 00010000:00000001:0.0:1788572471.870605:0:261364:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.870608:0:261364:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.870611:0:261364:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.870619:0:261364:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870626:0:261364:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.870630:0:261364:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.870632:0:261364:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.870666:0:261364:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.870668:0:261364:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.870671:0:261364:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdc2:0x0].0x0 (ffff9c372f514d00) refcount = 2 00010000:00000040:0.0:1788572471.870674:0:261364:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.870677:0:261364:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee40c00/0x1441586093167f4b lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a115e expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.870685:0:261364:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870692:0:261364:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.870696:0:261364:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.870698:0:261364:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.870701:0:261364:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.870704:0:261364:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.870707:0:261364:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870713:0:261364:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.870715:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870717:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.870719:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870720:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.870722:0:261364:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.870726:0:261364:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.870728:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870729:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870731:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.870733:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870735:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870737:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870738:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870740:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=4 00000020:00000001:0.0:1788572471.870741:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870743:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870745:0:261364:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.870748:0:261364:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.870751:0:261364:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870757:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870758:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.870759:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870760:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870761:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=5 00000020:00000001:0.0:1788572471.870763:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870764:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870766:0:261364:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.870769:0:261364:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870774:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870775:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.870777:0:261364:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870784:0:261364:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.870786:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870787:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.870788:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870789:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.870794:0:261364:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.870802:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870803:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870804:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.870806:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870808:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00000002:00010000:0.0:1788572471.870812:0:261364:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.870824:0:261364:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d0910c0. 00010000:00000001:0.0:1788572471.870827:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870828:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.870829:0:261364:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.870831:0:261364:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.870834:0:261364:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.870836:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870837:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870838:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.870840:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870842:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00000002:00010000:0.0:1788572471.870845:0:261364:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.870851:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870853:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.870854:0:261364:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.870856:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.870857:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.870858:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.870860:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.870862:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000040:0.0:1788572471.870865:0:261364:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514d00 count: 3 00010000:00000001:0.0:1788572471.870867:0:261364:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.870871:0:261364:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3522 (0 0) 00010000:00000001:0.0:1788572471.870873:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.870874:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.870875:0:261364:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.870878:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b710 00000002:00002000:0.0:1788572471.870891:0:261364:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-046 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.870894:0:261364:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.870898:0:261364:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.870900:0:261364:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.870903:0:261364:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.870906:0:261364:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.870910:0:261364:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.870912:0:261364:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.870914:0:261364:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc207bb20 it_disposition b 00000080:00000001:0.0:1788572471.870917:0:261364:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.870923:0:261364:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.870926:0:261364:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.870928:0:261364:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329090 : 144115373145329090 : 200002b17000dc2) 00000080:00000001:0.0:1788572471.870931:0:261364:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.870937:0:261364:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.870941:0:261364:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793672 : -109714643757944 : ffff9c3711a1d488) 00000020:00001000:0.0:1788572471.870944:0:261364:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d480 00000020:00000001:0.0:1788572471.870946:0:261364:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.870950:0:261364:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.870952:0:261364:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.870958:0:261364:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.870962:0:261364:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.870968:0:261364:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.870970:0:261364:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.870973:0:261364:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdc2:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.870978:0:261364:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.870982:0:261364:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21106 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.870985:0:261364:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4bba0) owner ffff9c37302ea7c0 00020000:00000001:0.0:1788572471.870988:0:261364:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.870990:0:261364:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4bba0) owner ffff9c37302ea7c0 00020000:00000010:0.0:1788572471.870994:0:261364:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.870998:0:261364:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.871000:0:261364:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.871002:0:261364:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc2:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.871005:0:261364:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871006:0:261364:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.871008:0:261364:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d480 00000080:00000001:0.0:1788572471.871012:0:261364:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329090 : 144115373145329090 : 200002b17000dc2) 00000080:00000001:0.0:1788572471.871014:0:261364:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.871019:0:261364:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdc2:0x0](ffff9c3708760908): rc = 0 00000080:00000001:0.0:1788572471.871022:0:261364:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911928584 : -109714797623032 : ffff9c3708760908) 00010000:00000001:0.0:1788572471.871024:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.871025:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.871026:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.871028:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871030:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00000080:00000001:0.0:1788572471.871033:0:261364:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.871034:0:261364:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.871036:0:261364:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793672 : -109714643757944 : ffff9c3711a1d488) 00000020:00001000:0.0:1788572471.871037:0:261364:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d480 00000020:00000001:0.0:1788572471.871039:0:261364:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.871040:0:261364:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.871041:0:261364:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.871044:0:261364:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.871046:0:261364:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.871049:0:261364:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.871050:0:261364:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.871052:0:261364:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdc2:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.871056:0:261364:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.871059:0:261364:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21106 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572471.871061:0:261364:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4bba0) owner ffff9c37302ea7c0 00020000:00000001:0.0:1788572471.871063:0:261364:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.871064:0:261364:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4bba0) owner ffff9c37302ea7c0 00020000:00000010:0.0:1788572471.871067:0:261364:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.871069:0:261364:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.871071:0:261364:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.871072:0:261364:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc2:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.871074:0:261364:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871075:0:261364:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871078:0:261364:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.871081:0:261364:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4bba0) owner 0000000000000000/ffff9c37302ea7c0 00020000:00000002:0.0:1788572471.871084:0:261364:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c371ec971e0 addref 2/0 by ffff9c37302ea7c0. 00020000:00000002:0.0:1788572471.871086:0:261364:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4bba0) owner 0000000000000000/ffff9c37302ea7c0 00020000:00000001:0.0:1788572471.871089:0:261364:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.871091:0:261364:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.871092:0:261364:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.871094:0:261364:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.871096:0:261364:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.871098:0:261364:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdc2:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.871100:0:261364:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d480 00000080:00000001:0.0:1788572471.871102:0:261364:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871103:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871104:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.871106:0:261364:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.871108:0:261364:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.871110:0:261364:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.871112:0:261364:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc2:0x0](ffff9c3708760908) for lock 0x144158609316856b 00800000:00000001:0.0:1788572471.871116:0:261364:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.871117:0:261364:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.871119:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.871120:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.871121:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.871123:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871125:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871127:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871128:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.871129:0:261364:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.871130:0:261364:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.871136:0:261364:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f9000 inode ffff9c3708760908 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.871140:0:261364:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.871143:0:261364:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc2:0x0](ffff9c3708760908) 00000080:00010000:0.0:1788572471.871146:0:261364:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316856b from it ffffb95bc207bb20 00010000:00000001:0.0:1788572471.871148:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.871149:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.871150:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.871152:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871154:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871156:0:261364:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.871159:0:261364:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.871166:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871167:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.871168:0:261364:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.871171:0:261364:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.871173:0:261364:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.871180:0:261364:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.871182:0:261364:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.871184:0:261364:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.871188:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871189:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.871190:0:261364:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871193:0:261364:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.871195:0:261364:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871197:0:261364:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871199:0:261364:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871201:0:261364:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.871202:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871203:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.871205:0:261364:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.871208:0:261364:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000100:00000001:0.0:1788572471.871214:0:261364:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.871219:0:261364:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809944192/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.871225:0:261364:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.871228:0:261364:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.871231:0:261364:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.871234:0:261364:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.871236:0:261364:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.871237:0:261364:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.871239:0:261364:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.871242:0:261364:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.871245:0:261364:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.871247:0:261364:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.871249:0:261364:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.871252:0:261364:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.871253:0:261364:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.871256:0:261364:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.871258:0:261364:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc207bb20 released 00000080:00000001:0.0:1788572471.871259:0:261364:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.871272:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00200008:0.0:1788572471.871274:0:261364:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-046:[0x200002b17:0xdc2:0x0](ffff9c3708760908), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.871280:0:261364:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.871282:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc80 00000080:00200000:0.0:1788572471.871283:0:261364:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdc2:0x0](ffff9c3708760908),name=f434.sanity-046 00000080:00000010:0.0:1788572471.871289:0:261364:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572471.871294:0:261364:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.871299:0:261364:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.871302:0:261364:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.871303:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bba0 00800000:00000002:0.0:1788572471.871305:0:261364:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdc2:0x0] 'f434.sanity-046' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.871308:0:261364:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.871312:0:261364:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.871314:0:261364:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.871316:0:261364:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.871319:0:261364:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.871321:0:261364:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.871322:0:261364:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.871325:0:261364:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.871327:0:261364:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.871328:0:261364:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.871329:0:261364:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.871330:0:261364:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.871332:0:261364:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.871333:0:261364:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdc2:0x0] 00800000:00000001:0.0:1788572471.871335:0:261364:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.871338:0:261364:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdc2:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.871342:0:261364:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.871343:0:261364:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba70 00000002:00010000:0.0:1788572471.871345:0:261364:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdc2:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.871349:0:261364:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.871350:0:261364:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.871352:0:261364:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.871355:0:261364:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514d00 count: 3 00010000:00000001:0.0:1788572471.871357:0:261364:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.871359:0:261364:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.871361:0:261364:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.871364:0:261364:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.871376:0:261364:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.871379:0:261364:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.871382:0:261364:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.871389:0:261364:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.871391:0:261364:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.871392:0:261364:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.871394:0:261364:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.871396:0:261364:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.871398:0:261364:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.871400:0:261364:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.871401:0:261364:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.871404:0:261364:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572471.871407:0:261364:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.871409:0:261364:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.871410:0:261364:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc2:0x0](ffff9c3708760908) 00000080:00010000:0.0:1788572471.871413:0:261364:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc2:0x0](ffff9c3708760908) for lock 0x144158609316856b 00800000:00000001:0.0:1788572471.871416:0:261364:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.871417:0:261364:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.871419:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.871420:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.871421:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.871423:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871425:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871427:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871428:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.871430:0:261364:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.871431:0:261364:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.871433:0:261364:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316856b from it ffffb95bc207bcb8 00010000:00000001:0.0:1788572471.871435:0:261364:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.871436:0:261364:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.871437:0:261364:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b200 refcount=3 00000020:00000001:0.0:1788572471.871439:0:261364:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871441:0:261364:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553700352 : -109714155851264 : ffff9c372eb6b200) 00010000:00000001:0.0:1788572471.871443:0:261364:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.871445:0:261364:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.871451:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871452:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.871453:0:261364:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.871454:0:261364:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.871456:0:261364:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6b200/0x144158609316856b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24b3 expref: -99 pid: 261364 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.871462:0:261364:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.871464:0:261364:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.871465:0:261364:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.871467:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871468:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.871469:0:261364:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871471:0:261364:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.871473:0:261364:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871474:0:261364:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871476:0:261364:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871478:0:261364:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.871479:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871480:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.871482:0:261364:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.871483:0:261364:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.871486:0:261364:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.871489:0:261364:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.871491:0:261364:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.871493:0:261364:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.871494:0:261364:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.871497:0:261364:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.871499:0:261364:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.871501:0:261364:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.871503:0:261364:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793672 : -109714643757944 : ffff9c3711a1d488) 00000020:00001000:0.0:1788572471.871505:0:261364:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d480 00000020:00000001:0.0:1788572471.871509:0:261364:0:(cl_object.c:143:cl_object_top()) ffff9c371f02ac38 -> ffff9c371f02ac38 00000020:00000001:0.0:1788572471.871511:0:261364:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.871513:0:261364:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.871515:0:261364:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdc2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.871518:0:261364:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.871519:0:261364:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.871521:0:261364:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.871522:0:261364:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.871524:0:261364:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.871527:0:261364:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdc2:0x0]io ffff9c3705179780 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.871530:0:261364:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4bba0) owner 0000000000000000/ffff9c37302ea7c0 00020000:00000001:0.0:1788572471.871533:0:261364:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.871534:0:261364:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.871536:0:261364:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.871538:0:261364:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871540:0:261364:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.871542:0:261364:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.871543:0:261364:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871544:0:261364:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.871545:0:261364:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.871546:0:261364:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.871547:0:261364:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4bba0) owner 0000000000000000/ffff9c37302ea7c0 00000020:00000001:0.0:1788572471.871549:0:261364:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.871551:0:261364:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.871552:0:261364:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdc2:0x0] 00000020:00000001:0.0:1788572471.871555:0:261364:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.871556:0:261364:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.871559:0:261364:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.871560:0:261364:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.871565:0:261364:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.871571:0:261364:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676280. 00020000:00000002:0.0:1788572471.871573:0:261364:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.871575:0:261364:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.871577:0:261364:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.871579:0:261364:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.871581:0:261364:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.871582:0:261364:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941928 : -109714522609688 : ffff9c3718da67e8) 00000020:00001000:0.0:1788572471.871584:0:261364:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da67e0 00000020:00000001:0.0:1788572471.871586:0:261364:0:(cl_object.c:143:cl_object_top()) ffff9c371f02a468 -> ffff9c371f02ac38 00000020:00000001:0.0:1788572471.871587:0:261364:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.871590:0:261364:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.871591:0:261364:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.871592:0:261364:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871594:0:261364:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871595:0:261364:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859897800 : -109714849653816 : ffff9c37055c1bc8) 00000020:00000001:0.0:1788572471.871597:0:261364:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.871601:0:261364:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a7430. 00000008:00010000:0.0:1788572471.871604:0:261364:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.871606:0:261364:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.871607:0:261364:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.871609:0:261364:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d6762b0, osc lock ffff9c37030a7430, flags 0x400001002 00000020:00000001:0.0:1788572471.871611:0:261364:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871613:0:261364:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871615:0:261364:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725440 : -109714177826176 : ffff9c372d676280) 00000020:00000001:0.0:1788572471.871617:0:261364:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.871618:0:261364:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.871620:0:261364:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871621:0:261364:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871624:0:261364:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.871625:0:261364:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.871626:0:261364:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.871628:0:261364:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.871629:0:261364:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.871630:0:261364:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859897800 : -109714849653816 : ffff9c37055c1bc8) 00000020:00000001:0.0:1788572471.871632:0:261364:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.871720:0:261364:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.871723:0:261364:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.871727:0:261364:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.871729:0:261364:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.871730:0:261364:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.871732:0:261364:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.871734:0:261364:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.871735:0:261364:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.871737:0:261364:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.871740:0:261364:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.871744:0:261364:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871746:0:261364:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.871750:0:261364:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.871753:0:261364:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.871757:0:261364:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.871762:0:261364:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.871765:0:261364:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.871767:0:261364:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.871768:0:261364:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.871770:0:261364:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.871772:0:261364:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.871775:0:261364:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.871779:0:261364:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7200. 00000100:00000001:0.0:1788572471.871785:0:261364:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871788:0:261364:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.871789:0:261364:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871791:0:261364:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.871792:0:261364:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871794:0:261364:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.871797:0:261364:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.871799:0:261364:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.871801:0:261364:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.871804:0:261364:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0900. 00010000:00000010:0.0:1788572471.871807:0:261364:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0400. 00010000:00000001:0.0:1788572471.871810:0:261364:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.871812:0:261364:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6ae00. 00000020:00000001:0.0:1788572471.871816:0:261364:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.871818:0:261364:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6ae00 with handle 0x1441586093168572 to hash 00000020:00000001:0.0:1788572471.871820:0:261364:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.871821:0:261364:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000010:0.0:1788572471.871824:0:261364:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091ac0. 00010000:00000001:0.0:1788572471.871827:0:261364:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.871829:0:261364:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.871831:0:261364:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.871834:0:261364:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.871844:0:261364:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.871855:0:261364:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.871859:0:261364:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.871863:0:261364:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.871870:0:261364:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.871911:0:261364:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.871914:0:261364:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871916:0:261364:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.871918:0:261364:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871919:0:261364:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.871922:0:261364:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.871924:0:261364:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.871926:0:261364:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.871927:0:261364:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572471.871939:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572471.871943:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572471.871944:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.871951:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.871953:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.871955:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.871957:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.871959:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.871965:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.872601:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.872603:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.872607:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.872610:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.872613:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.872617:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809944320:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.872621:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.872627:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.872629:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.872631:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.872658:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.872661:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.872668:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.872672:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.872679:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.872686:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809944320, portal 4 00000100:00000001:0.0:1788572471.872688:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.872689:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572471.872696:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.872704:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.872707:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.872709:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572471.872712:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809944320, offset 0 00000400:00000200:0.0:1788572471.872718:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.872725:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.872732:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.872737:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.872741:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.872745:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.872747:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.872751:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539757:539757:256:4294967295] 192.168.202.148@tcp LPNI seq info [539722:539722:8:4294967295] 00000400:00000200:0.0:1788572471.872755:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.872763:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.872769:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.872776:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788572471.872783:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.872790:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.872794:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0b00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.872801:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.872806:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 169390 usecs (170 jiffies) 00000400:00000200:0.0:1788572471.872809:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.872811:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307611048 00000100:00000001:0.0:1788572471.872814:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.872816:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307611048 00000100:00000001:0.0:1788572471.872819:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.872821:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.872824:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.872827:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.872829:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.884983:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.884988:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.884995:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.884998:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.885002:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.885006:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.885009:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.885012:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.885018:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.885027:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.885034:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.885036:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.885040:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.885044:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.885046:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.885049:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.885052:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.885063:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.885068:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=86 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572471.885071:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.885073:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885076:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.885081:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.885085:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.885087:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572471.885089:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.885092:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.885097:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.885105:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.885108:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.885111:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.885114:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.885117:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885120:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885124:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=4 00000020:00000001:0.0:1788572471.885126:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885129:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885132:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885133:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885134:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=5 00000020:00000001:0.0:1788572471.885136:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885138:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885142:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.885145:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.885150:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885167:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885169:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.885172:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.885174:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885175:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885176:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=6 00000020:00000001:0.0:1788572471.885178:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885180:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000040:0.0:1788572471.885186:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6ae00, remote cookie: 0xd9894672d99a24ba, flags: 0x1001 00010000:00010000:0.0:1788572471.885190:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885199:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.885202:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885205:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.885209:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.885215:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.885223:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x5272:0x0].0x0 (ffff9c37198c0900) refcount = 1 00010000:00000040:0.0:1788572471.885225:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.885228:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885240:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.885242:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.885246:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.885248:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.885252:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885259:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.885260:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885261:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.885262:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885264:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.885266:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.885270:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.885274:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885275:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885276:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=5 00000020:00000001:0.0:1788572471.885278:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885280:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00000008:00000001:0.0:1788572471.885284:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.885289:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7adc00 -> ffff9c371f02ac38 00000008:00010000:0.0:1788572471.885292:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.885300:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7adc00 -> ffff9c371f02ac38 00000020:00000001:0.0:1788572471.885301:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.885304:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.885307:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.885309:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.885311:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.885313:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c372d7adc00 -> ffff9c371f02ac38 00000008:00000001:0.0:1788572471.885314:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.885316:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.885318:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.885336:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.885339:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885341:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885342:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885343:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=6 00000020:00000001:0.0:1788572471.885345:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885347:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885349:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.885351:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885357:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885359:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.885361:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885367:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.885368:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885369:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.885370:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885372:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.885373:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.885374:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6ae00 refcount=5 00000020:00000001:0.0:1788572471.885376:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885377:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553699328 : -109714155852288 : ffff9c372eb6ae00) 00010000:00000001:0.0:1788572471.885379:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.885381:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.885387:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885388:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.885389:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.885392:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.885394:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000579bc4da/0x1441586093168572 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x5272:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a24ba expref: -99 pid: 261364 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.885400:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.885403:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.885406:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.885409:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885410:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.885412:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885414:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.885416:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885418:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885420:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.885421:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.885423:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885423:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.885425:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.885425:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.885427:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.885432:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.885441:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809944320:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.885447:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.885449:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.885453:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809944320/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.885460:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.885462:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.885465:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.885469:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.885471:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.885473:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.885475:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.885478:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572471.885483:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.885485:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.885487:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.885490:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.885491:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.885494:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.885500:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.885501:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.885504:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.885506:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.915004:0:261366:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572471.915021:0:261366:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.915031:0:261366:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.915033:0:261366:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.915034:0:261366:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.915035:0:261366:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.915037:0:261366:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.915049:0:261366:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00010000:00000001:0.0:1788572471.915054:0:261366:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.915057:0:261366:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572471.915064:0:261366:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572471.915067:0:261366:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.915070:0:261366:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.915076:0:261366:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6a800. 00000020:00000001:0.0:1788572471.915081:0:261366:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.915084:0:261366:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6a800 with handle 0x1441586093168579 to hash 00000020:00000001:0.0:1788572471.915087:0:261366:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.915088:0:261366:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.915092:0:261366:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.915095:0:261366:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.915098:0:261366:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.915102:0:261366:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261366 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.915112:0:261366:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261366 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.915122:0:261366:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261366 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572471.915133:0:261366:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572471.915135:0:261366:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572471.915143:0:261366:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.915145:0:261366:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572471.915148:0:261366:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.915149:0:261366:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000001:0.0:1788572471.915154:0:261366:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.915158:0:261366:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.915160:0:261366:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572471.915162:0:261366:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572471.915173:0:261366:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.915183:0:261366:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572471.915185:0:261366:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.915189:0:261366:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572471.915191:0:261366:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572471.915194:0:261366:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572471.915198:0:261366:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261366:1875440809944448:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.915201:0:261366:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572471.915206:0:261366:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572471.915208:0:261366:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.915211:0:261366:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572471.915216:0:261366:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.915218:0:261366:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572471.915226:0:261366:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572471.915230:0:261366:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572471.915236:0:261366:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572471.915243:0:261366:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809944448, portal 10 00000100:00000001:0.0:1788572471.915244:0:261366:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572471.915246:0:261366:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000040:0.0:1788572471.915252:0:261366:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.915264:0:261366:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572471.915267:0:261366:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572471.915269:0:261366:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572471.915272:0:261366:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809944448, offset 0 00000400:00000200:0.0:1788572471.915278:0:261366:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572471.915287:0:261366:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.915295:0:261366:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572471.915301:0:261366:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572471.915304:0:261366:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572471.915308:0:261366:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572471.915311:0:261366:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572471.915315:0:261366:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539758:539758:256:4294967295] 192.168.202.148@tcp LPNI seq info [539723:539723:8:4294967295] 00000400:00000200:0.0:1788572471.915320:0:261366:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572471.915330:0:261366:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572471.915336:0:261366:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572471.915341:0:261366:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788572471.915346:0:261366:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572471.915354:0:261366:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572471.915358:0:261366:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0700 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572471.915374:0:261366:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.915378:0:261366:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14987 usecs (15 jiffies) 00000400:00000200:0.0:1788572471.915381:0:261366:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572471.915384:0:261366:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307610936 00000100:00000001:0.0:1788572471.915387:0:261366:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.915389:0:261366:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307610936 00000100:00000001:0.0:1788572471.915391:0:261366:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572471.915393:0:261366:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.915396:0:261366:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.915399:0:261366:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.915401:0:261366:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572471.915403:0:261366:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572471.915406:0:261366:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572471.915409:0:261366:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.922709:0:261366:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.922718:0:261366:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.922720:0:261366:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.922724:0:261366:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c89a0. 00000400:00000200:0.0:1788572471.922729:0:261366:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572471.922731:0:261366:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572471.922735:0:261366:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.922739:0:261366:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.922748:0:261366:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.922754:0:261366:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.922756:0:261366:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.922759:0:261366:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.922762:0:261366:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.922764:0:261366:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.922766:0:261366:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.922769:0:261366:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.922779:0:261366:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=35 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572471.922783:0:261366:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=67 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572471.922785:0:261366:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.922787:0:261366:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.922790:0:261366:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.922794:0:261366:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.922800:0:261366:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.922801:0:261366:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572471.922804:0:261366:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.922807:0:261366:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.922812:0:261366:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.922837:0:261366:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.922840:0:261366:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.922842:0:261366:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572471.922847:0:261366:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.922854:0:261366:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261366:1875440809944448:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.922859:0:261366:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.922863:0:261366:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.922865:0:261366:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572471.922867:0:261366:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.922870:0:261366:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.922877:0:261366:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572471.922879:0:261366:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572471.922891:0:261366:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572471.922892:0:261366:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.922895:0:261366:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.922901:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.922903:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.922907:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=4 00000020:00000001:0.0:1788572471.922909:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.922912:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000040:0.0:1788572471.922920:0:261366:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6a800, remote cookie: 0xd9894672d99a24d6, flags: 0x1001 00010000:00010000:0.0:1788572471.922926:0:261366:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.922936:0:261366:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdc3:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572471.922939:0:261366:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572471.922947:0:261366:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3707e6a200 count: 2 00010000:00000001:0.0:1788572471.922950:0:261366:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.922953:0:261366:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.922957:0:261366:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.922964:0:261366:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.922970:0:261366:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.922974:0:261366:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572471.922976:0:261366:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572471.922978:0:261366:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572471.922979:0:261366:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572471.922982:0:261366:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdc3:0x0].0x0 (ffff9c3707e6a200) refcount = 2 00010000:00000040:0.0:1788572471.922985:0:261366:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.922988:0:261366:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371fd1a400/0x1441586093167f59 lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a118f expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572471.922995:0:261366:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923002:0:261366:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572471.923006:0:261366:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.923008:0:261366:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.923011:0:261366:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.923013:0:261366:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.923017:0:261366:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923022:0:261366:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.923024:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923025:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923026:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923028:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923030:0:261366:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.923034:0:261366:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572471.923036:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923037:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923039:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923040:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923042:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923044:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923045:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923046:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=4 00000020:00000001:0.0:1788572471.923048:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923050:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923052:0:261366:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.923056:0:261366:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.923059:0:261366:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923066:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923067:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923068:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923069:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923071:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=5 00000020:00000001:0.0:1788572471.923072:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923074:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923076:0:261366:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.923079:0:261366:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923085:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923086:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.923088:0:261366:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923099:0:261366:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.923101:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923102:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923103:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923104:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572471.923109:0:261366:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572471.923119:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923120:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923121:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923123:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923125:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00000002:00010000:0.0:1788572471.923129:0:261366:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572471.923139:0:261366:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091b40. 00010000:00000001:0.0:1788572471.923142:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923143:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.923145:0:261366:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.923146:0:261366:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572471.923150:0:261366:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572471.923152:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923153:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923155:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923156:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923158:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00000002:00010000:0.0:1788572471.923161:0:261366:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923168:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923169:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923171:0:261366:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.923172:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923173:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923175:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923176:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923178:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000040:0.0:1788572471.923181:0:261366:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3707e6a200 count: 3 00010000:00000001:0.0:1788572471.923183:0:261366:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572471.923187:0:261366:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3523 (0 0) 00010000:00000001:0.0:1788572471.923189:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923190:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.923191:0:261366:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572471.923195:0:261366:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab710 00000002:00002000:0.0:1788572471.923197:0:261366:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-047 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572471.923200:0:261366:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.923205:0:261366:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.923208:0:261366:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572471.923211:0:261366:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572471.923214:0:261366:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.923218:0:261366:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.923221:0:261366:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572471.923223:0:261366:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc20abb20 it_disposition b 00000080:00000001:0.0:1788572471.923226:0:261366:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572471.923234:0:261366:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572471.923237:0:261366:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572471.923240:0:261366:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329091 : 144115373145329091 : 200002b17000dc3) 00000080:00000001:0.0:1788572471.923243:0:261366:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572471.923249:0:261366:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.923254:0:261366:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943224 : -109714522608392 : ffff9c3718da6cf8) 00000020:00001000:0.0:1788572471.923256:0:261366:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6cf0 00000020:00000001:0.0:1788572471.923259:0:261366:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.923262:0:261366:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.923265:0:261366:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.923271:0:261366:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.923275:0:261366:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.923281:0:261366:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572471.923285:0:261366:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572471.923288:0:261366:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdc3:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.923293:0:261366:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.923297:0:261366:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20992 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.923302:0:261366:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c372d066d90) owner ffff9c372d1c0000 00020000:00000001:0.0:1788572471.923304:0:261366:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.923306:0:261366:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c372d066d90) owner ffff9c372d1c0000 00020000:00000010:0.0:1788572471.923311:0:261366:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.923315:0:261366:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572471.923319:0:261366:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572471.923321:0:261366:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc3:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.923322:0:261366:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.923324:0:261366:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572471.923326:0:261366:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6cf0 00000080:00000001:0.0:1788572471.923330:0:261366:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329091 : 144115373145329091 : 200002b17000dc3) 00000080:00000001:0.0:1788572471.923333:0:261366:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572471.923338:0:261366:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdc3:0x0](ffff9c37087ba1c8): rc = 0 00000080:00000001:0.0:1788572471.923341:0:261366:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358912295368 : -109714797256248 : ffff9c37087ba1c8) 00010000:00000001:0.0:1788572471.923343:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923344:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923346:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923347:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923349:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00000080:00000001:0.0:1788572471.923353:0:261366:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572471.923354:0:261366:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.923356:0:261366:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943224 : -109714522608392 : ffff9c3718da6cf8) 00000020:00001000:0.0:1788572471.923358:0:261366:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6cf0 00000020:00000001:0.0:1788572471.923359:0:261366:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572471.923361:0:261366:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572471.923362:0:261366:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572471.923365:0:261366:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.923368:0:261366:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.923370:0:261366:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572471.923371:0:261366:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572471.923373:0:261366:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdc3:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572471.923377:0:261366:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572471.923380:0:261366:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20992 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572471.923382:0:261366:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c372d066d90) owner ffff9c372d1c0000 00020000:00000001:0.0:1788572471.923384:0:261366:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572471.923385:0:261366:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c372d066d90) owner ffff9c372d1c0000 00020000:00000010:0.0:1788572471.923387:0:261366:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572471.923389:0:261366:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572471.923391:0:261366:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572471.923393:0:261366:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdc3:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572471.923394:0:261366:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.923396:0:261366:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.923399:0:261366:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572471.923402:0:261366:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c372d066d90) owner 0000000000000000/ffff9c372d1c0000 00020000:00000002:0.0:1788572471.923405:0:261366:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3731cf6180 addref 2/0 by ffff9c372d1c0000. 00020000:00000002:0.0:1788572471.923407:0:261366:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c372d066d90) owner 0000000000000000/ffff9c372d1c0000 00020000:00000001:0.0:1788572471.923410:0:261366:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572471.923412:0:261366:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572471.923413:0:261366:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572471.923415:0:261366:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572471.923418:0:261366:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572471.923420:0:261366:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdc3:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572471.923422:0:261366:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6cf0 00000080:00000001:0.0:1788572471.923425:0:261366:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.923427:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923428:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.923430:0:261366:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572471.923433:0:261366:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572471.923435:0:261366:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.923439:0:261366:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc3:0x0](ffff9c37087ba1c8) for lock 0x1441586093168579 00800000:00000001:0.0:1788572471.923443:0:261366:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.923445:0:261366:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.923446:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923448:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923449:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923451:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923454:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923456:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923458:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.923459:0:261366:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.923462:0:261366:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572471.923471:0:261366:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37061cc000 inode ffff9c37087ba1c8 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572471.923475:0:261366:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572471.923480:0:261366:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc3:0x0](ffff9c37087ba1c8) 00000080:00010000:0.0:1788572471.923485:0:261366:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168579 from it ffffb95bc20abb20 00010000:00000001:0.0:1788572471.923487:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.923488:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.923489:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.923491:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923492:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.923494:0:261366:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.923497:0:261366:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.923503:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923504:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923505:0:261366:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.923508:0:261366:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.923510:0:261366:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.923518:0:261366:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.923521:0:261366:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.923524:0:261366:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.923529:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923530:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.923533:0:261366:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.923536:0:261366:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.923539:0:261366:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.923541:0:261366:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.923544:0:261366:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.923546:0:261366:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.923547:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.923548:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572471.923550:0:261366:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572471.923554:0:261366:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711214600. 00000100:00000001:0.0:1788572471.923559:0:261366:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.923564:0:261366:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440809944448/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572556 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.923571:0:261366:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.923574:0:261366:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.923577:0:261366:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572471.923581:0:261366:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.923583:0:261366:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.923585:0:261366:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572471.923587:0:261366:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.923590:0:261366:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572471.923593:0:261366:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.923595:0:261366:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.923597:0:261366:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788572471.923601:0:261366:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.923602:0:261366:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.923605:0:261366:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572471.923607:0:261366:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc20abb20 released 00000080:00000001:0.0:1788572471.923608:0:261366:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572471.923621:0:261366:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abd08 00000080:00200008:0.0:1788572471.923624:0:261366:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-047:[0x200002b17:0xdc3:0x0](ffff9c37087ba1c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.923630:0:261366:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.923632:0:261366:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abc80 00000080:00200000:0.0:1788572471.924295:0:261366:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdc3:0x0](ffff9c37087ba1c8),name=f434.sanity-047 00000080:00000010:0.0:1788572471.924308:0:261366:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6800. 00800000:00000002:0.0:1788572471.924314:0:261366:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572471.924321:0:261366:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.924325:0:261366:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.924327:0:261366:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abba0 00800000:00000002:0.0:1788572471.924329:0:261366:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdc3:0x0] 'f434.sanity-047' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.924332:0:261366:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.924336:0:261366:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.924339:0:261366:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.924342:0:261366:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.924345:0:261366:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.924347:0:261366:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.924349:0:261366:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572471.924353:0:261366:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.924356:0:261366:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.924357:0:261366:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.924358:0:261366:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.924359:0:261366:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.924361:0:261366:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.924363:0:261366:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdc3:0x0] 00800000:00000001:0.0:1788572471.924365:0:261366:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.924367:0:261366:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdc3:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.924372:0:261366:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.924373:0:261366:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20aba70 00000002:00010000:0.0:1788572471.924375:0:261366:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdc3:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.924379:0:261366:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.924381:0:261366:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.924383:0:261366:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.924388:0:261366:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3707e6a200 count: 3 00010000:00000001:0.0:1788572471.924391:0:261366:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.924394:0:261366:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.924398:0:261366:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.924401:0:261366:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572471.924416:0:261366:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.924420:0:261366:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.924424:0:261366:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572471.924430:0:261366:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.924432:0:261366:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.924434:0:261366:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.924436:0:261366:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.924438:0:261366:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.924440:0:261366:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572471.924442:0:261366:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.924445:0:261366:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572471.924448:0:261366:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6800. 00000080:00000001:0.0:1788572471.924454:0:261366:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.924456:0:261366:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.924459:0:261366:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdc3:0x0](ffff9c37087ba1c8) 00000080:00010000:0.0:1788572471.924462:0:261366:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdc3:0x0](ffff9c37087ba1c8) for lock 0x1441586093168579 00800000:00000001:0.0:1788572471.924465:0:261366:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.924467:0:261366:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.924468:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.924470:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.924471:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.924474:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.924476:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.924479:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.924480:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.924481:0:261366:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.924482:0:261366:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.924485:0:261366:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168579 from it ffffb95bc20abcb8 00010000:00000001:0.0:1788572471.924486:0:261366:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.924488:0:261366:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.924489:0:261366:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6a800 refcount=3 00000020:00000001:0.0:1788572471.924491:0:261366:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.924493:0:261366:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553697792 : -109714155853824 : ffff9c372eb6a800) 00010000:00000001:0.0:1788572471.924496:0:261366:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.924498:0:261366:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.924505:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.924506:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.924507:0:261366:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.924508:0:261366:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.924511:0:261366:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372eb6a800/0x1441586093168579 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a24d6 expref: -99 pid: 261366 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572471.924518:0:261366:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.924520:0:261366:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.924521:0:261366:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.924524:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.924525:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.924527:0:261366:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924529:0:261366:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.924531:0:261366:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924533:0:261366:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924535:0:261366:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924537:0:261366:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.924538:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.924539:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.924542:0:261366:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.924544:0:261366:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.924547:0:261366:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572471.924550:0:261366:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.924552:0:261366:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572471.924554:0:261366:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572471.924556:0:261366:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.924559:0:261366:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.924562:0:261366:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572471.924565:0:261366:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.924568:0:261366:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943224 : -109714522608392 : ffff9c3718da6cf8) 00000020:00001000:0.0:1788572471.924570:0:261366:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6cf0 00000020:00000001:0.0:1788572471.924575:0:261366:0:(cl_object.c:143:cl_object_top()) ffff9c37031ffe90 -> ffff9c37031ffe90 00000020:00000001:0.0:1788572471.924578:0:261366:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572471.924580:0:261366:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572471.924582:0:261366:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdc3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572471.924585:0:261366:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.924587:0:261366:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572471.924588:0:261366:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.924591:0:261366:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.924593:0:261366:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.924597:0:261366:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdc3:0x0]io ffff9c371ff50060 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572471.924600:0:261366:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c372d066d90) owner 0000000000000000/ffff9c372d1c0000 00020000:00000001:0.0:1788572471.924604:0:261366:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572471.924605:0:261366:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572471.924607:0:261366:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572471.924609:0:261366:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924611:0:261366:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572471.924613:0:261366:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572471.924615:0:261366:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924617:0:261366:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.924618:0:261366:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.924619:0:261366:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572471.924621:0:261366:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c372d066d90) owner 0000000000000000/ffff9c372d1c0000 00000020:00000001:0.0:1788572471.924623:0:261366:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572471.924626:0:261366:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572471.924627:0:261366:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdc3:0x0] 00000020:00000001:0.0:1788572471.924630:0:261366:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572471.924632:0:261366:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572471.924687:0:261366:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572471.924689:0:261366:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572471.924694:0:261366:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572471.924700:0:261366:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676000. 00020000:00000002:0.0:1788572471.924702:0:261366:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572471.924704:0:261366:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572471.924706:0:261366:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.924708:0:261366:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572471.924709:0:261366:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572471.924711:0:261366:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941784 : -109714522609832 : ffff9c3718da6758) 00000020:00001000:0.0:1788572471.924713:0:261366:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6750 00000020:00000001:0.0:1788572471.924715:0:261366:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff918 -> ffff9c37031ffe90 00000020:00000001:0.0:1788572471.924717:0:261366:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572471.924720:0:261366:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572471.924721:0:261366:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572471.924723:0:261366:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924724:0:261366:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924726:0:261366:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293059160 : -109714416492456 : ffff9c371f2da058) 00000020:00000001:0.0:1788572471.924728:0:261366:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572471.924732:0:261366:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6720. 00000008:00010000:0.0:1788572471.924735:0:261366:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572471.924737:0:261366:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.924738:0:261366:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572471.924740:0:261366:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676030, osc lock ffff9c37030a6720, flags 0x400001002 00000020:00000001:0.0:1788572471.924742:0:261366:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924744:0:261366:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924746:0:261366:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531724800 : -109714177826816 : ffff9c372d676000) 00000020:00000001:0.0:1788572471.924749:0:261366:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572471.924750:0:261366:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572471.924751:0:261366:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924753:0:261366:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924755:0:261366:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572471.924756:0:261366:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572471.924758:0:261366:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.924759:0:261366:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572471.924760:0:261366:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.924761:0:261366:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293059160 : -109714416492456 : ffff9c371f2da058) 00000020:00000001:0.0:1788572471.924763:0:261366:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.924765:0:261366:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572471.924766:0:261366:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572471.924770:0:261366:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572471.924772:0:261366:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.924773:0:261366:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572471.924774:0:261366:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572471.924775:0:261366:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572471.924776:0:261366:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572471.924779:0:261366:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572471.924780:0:261366:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572471.924784:0:261366:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924786:0:261366:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572471.924790:0:261366:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572471.924794:0:261366:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572471.924798:0:261366:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572471.924802:0:261366:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.924805:0:261366:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.924806:0:261366:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.924807:0:261366:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.924809:0:261366:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572471.924811:0:261366:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572471.924814:0:261366:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572471.924831:0:261366:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6800. 00000100:00000001:0.0:1788572471.924837:0:261366:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924842:0:261366:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.924844:0:261366:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924845:0:261366:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572471.924846:0:261366:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924848:0:261366:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.924851:0:261366:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572471.924854:0:261366:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572471.924856:0:261366:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572471.924860:0:261366:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0a00. 00010000:00000010:0.0:1788572471.924864:0:261366:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0700. 00010000:00000001:0.0:1788572471.924868:0:261366:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572471.924870:0:261366:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372eb6b600. 00000020:00000001:0.0:1788572471.924875:0:261366:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572471.924877:0:261366:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372eb6b600 with handle 0x1441586093168580 to hash 00000020:00000001:0.0:1788572471.924879:0:261366:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572471.924880:0:261366:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000010:0.0:1788572471.924894:0:261366:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091a40. 00010000:00000001:0.0:1788572471.924896:0:261366:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.924898:0:261366:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.924900:0:261366:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.924904:0:261366:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572471.924913:0:261366:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.924922:0:261366:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.924926:0:261366:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572471.924930:0:261366:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.924939:0:261366:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572471.924963:0:261366:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572471.924965:0:261366:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924967:0:261366:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.924969:0:261366:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924971:0:261366:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.924973:0:261366:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.924974:0:261366:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572471.924976:0:261366:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572471.924977:0:261366:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000800:00000200:0.0:1788572471.925555:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572471.925559:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572471.925564:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572471.925570:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572471.925575:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f34c8 00000400:00000010:0.0:1788572471.925578:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f34c8. 00000100:00000001:0.0:1788572471.925582:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572471.925586:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.925597:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.925601:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.925613:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.925615:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572471.934548:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.934563:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572471.934572:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572471.934574:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572471.934578:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2210. 00000400:00000200:0.0:1788572471.934583:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3a18 00000400:00000010:0.0:1788572471.934587:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3a18. 00000100:00000001:0.0:1788572471.934590:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572471.934596:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572471.934605:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.934611:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572471.934613:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.934616:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572471.934619:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572471.934621:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572471.934624:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572471.934628:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572471.934687:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=35 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572471.934694:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=8 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572471.934696:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572471.934699:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.934702:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572471.934706:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.934710:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572471.934713:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572471.934715:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572471.934718:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.934722:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.934730:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572471.934734:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572471.934736:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572471.934739:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572471.934743:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.934746:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.934750:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=4 00000020:00000001:0.0:1788572471.934752:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.934755:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.934757:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.934758:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.934759:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=5 00000020:00000001:0.0:1788572471.934761:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.934763:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.934767:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.934770:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.934775:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.934787:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.934789:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.934792:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572471.934794:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.934796:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.934797:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=6 00000020:00000001:0.0:1788572471.934799:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.934801:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000040:0.0:1788572471.934808:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372eb6b600, remote cookie: 0xd9894672d99a24dd, flags: 0x1001 00010000:00010000:0.0:1788572471.934812:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.934991:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572471.934995:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.934998:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572471.935001:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572471.935005:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572471.935015:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x5200:0x0].0x0 (ffff9c37198c0a00) refcount = 1 00010000:00000040:0.0:1788572471.935017:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572471.935019:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.935029:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572471.935030:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.935033:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572471.935036:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.935039:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.935045:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572471.935046:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935047:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.935048:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935049:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.935051:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572471.935054:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572471.935057:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.935058:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.935060:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=5 00000020:00000001:0.0:1788572471.935062:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.935064:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00000008:00000001:0.0:1788572471.935068:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572471.935072:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3705331340 -> ffff9c37031ffe90 00000008:00010000:0.0:1788572471.935147:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572471.935157:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3705331340 -> ffff9c37031ffe90 00000020:00000001:0.0:1788572471.935159:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572471.935163:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572471.935167:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572471.935169:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.935171:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.935173:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3705331340 -> ffff9c37031ffe90 00000008:00000001:0.0:1788572471.935174:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572471.935177:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572471.935179:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572471.935202:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572471.935204:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935207:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.935208:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.935209:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=6 00000020:00000001:0.0:1788572471.935211:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.935213:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.935216:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.935218:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.935225:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935226:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572471.935228:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.935235:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.935236:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935237:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.935238:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935240:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.935241:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.935243:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372eb6b600 refcount=5 00000020:00000001:0.0:1788572471.935245:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.935247:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359553701376 : -109714155850240 : ffff9c372eb6b600) 00010000:00000001:0.0:1788572471.935248:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.935251:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572471.935257:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935258:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.935259:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.935262:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.935264:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c99832fe/0x1441586093168580 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a24dd expref: -99 pid: 261366 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572471.935272:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.935276:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.935278:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.935281:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935282:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.935284:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935287:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.935289:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935291:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935294:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.935296:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.935297:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935298:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.935300:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.935301:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.935302:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572471.935307:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572471.935315:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809944576:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572471.935320:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.935323:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572471.935327:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440809944576/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572487 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572471.935332:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572471.935335:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572471.935338:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe96400. 02000000:00000001:0.0:1788572471.935342:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572471.935344:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572471.935346:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572471.935348:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572471.935351:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b6800. 02000000:00000001:0.0:1788572471.935358:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572471.935359:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572471.935361:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788572471.935365:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572471.935366:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.935368:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572471.935375:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572471.935377:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572471.935379:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572471.935382:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979162:0:261368:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979175:0:261368:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.979182:0:261368:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572471.979187:0:261368:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff6c0 -> ffff9c37031ff6c0 00000020:00000001:0.0:1788572471.979190:0:261368:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff6c0 -> ffff9c37031ff6c0 00000020:00000001:0.0:1788572471.979192:0:261368:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572471.979197:0:261368:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572471.979203:0:261368:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.979205:0:261368:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572471.979206:0:261368:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572471.979207:0:261368:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.979209:0:261368:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdc4:11031 on OST[0]: s=0 (1) m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572471.979214:0:261368:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572471.979217:0:261368:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572471.979220:0:261368:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979222:0:261368:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979223:0:261368:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff6c0 -> ffff9c37031ff6c0 00000080:00200000:0.0:1788572471.979226:0:261368:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdc4:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572471.979228:0:261368:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572471.979231:0:261368:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572471.979233:0:261368:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731325030 (ffff9c3707076128/0) at cl_lock_release():245 00000020:00000001:0.0:1788572471.979236:0:261368:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.979238:0:261368:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731325030 (ffff9c3707076128/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572471.979241:0:261368:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572471.979245:0:261368:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572471.979247:0:261368:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314392296 : -109714395159320 : ffff9c37207324e8) 00000020:00000001:0.0:1788572471.979249:0:261368:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572471.979250:0:261368:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3720e47d30 (ffff9c3707076e18/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572471.979254:0:261368:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572471.979255:0:261368:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572471.979258:0:261368:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.979261:0:261368:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572471.979262:0:261368:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572471.979264:0:261368:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.979266:0:261368:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.979267:0:261368:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572471.979268:0:261368:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.979270:0:261368:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731325030 (ffff9c3707076128/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572471.979273:0:261368:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572471.979274:0:261368:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572471.979275:0:261368:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3720e47d30 (ffff9c3707076128/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572471.979279:0:261368:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3718db37c0. 00000020:00000001:0.0:1788572471.979285:0:261368:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572471.979287:0:261368:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c3720e47d00. 00020000:00000001:0.0:1788572471.979290:0:261368:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.979291:0:261368:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572471.979293:0:261368:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572471.979294:0:261368:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979297:0:261368:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572471.979301:0:261368:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572471.979302:0:261368:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572471.979304:0:261368:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572471.979307:0:261368:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.979309:0:261368:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076e10 00020000:00000001:0.0:1788572471.979315:0:261368:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572471.979318:0:261368:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572471.979321:0:261368:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572471.979324:0:261368:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdc4:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572471.979328:0:261368:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572471.979331:0:261368:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.979333:0:261368:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572471.979335:0:261368:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572471.979336:0:261368:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572471.979337:0:261368:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076120 00000001:00000001:0.0:1788572471.979339:0:261368:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.979351:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbd08 00000080:00000008:0.0:1788572471.979353:0:261368:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-048:[0x200002b17:0xdc4:0x0](ffff9c37087bc2c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572471.979360:0:261368:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572471.979362:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbda0 00000080:00002000:0.0:1788572471.979364:0:261368:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-048 (ffff9c3702109ca8, parent ffff9c3719d93af8, inode ffff9c37087bc2c8) hashed, 00000080:00000001:0.0:1788572471.979367:0:261368:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.979403:0:261368:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.979407:0:261368:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572471.979409:0:261368:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572471.979410:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbbc0 00000080:00200000:0.0:1788572471.979412:0:261368:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572471.979418:0:261368:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572471.979425:0:261368:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572471.979429:0:261368:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572471.979433:0:261368:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.979435:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbae0 00800000:00000002:0.0:1788572471.979437:0:261368:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.979442:0:261368:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572471.979445:0:261368:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572471.979448:0:261368:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572471.979450:0:261368:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572471.979453:0:261368:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572471.979455:0:261368:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.979457:0:261368:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572471.979459:0:261368:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572471.979466:0:261368:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572471.979471:0:261368:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572471.979473:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bb9b0 00000002:00010000:0.0:1788572471.979475:0:261368:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572471.979478:0:261368:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572471.979480:0:261368:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572471.979484:0:261368:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572471.979493:0:261368:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572471.979497:0:261368:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572471.979502:0:261368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572471.979507:0:261368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572471.979512:0:261368:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 02000000:00000001:0.0:1788572471.979523:0:261368:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572471.979529:0:261368:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572471.979533:0:261368:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 00000002:00000001:0.0:1788572471.979540:0:261368:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572471.979542:0:261368:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572471.979544:0:261368:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.979545:0:261368:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572471.979547:0:261368:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.979549:0:261368:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572471.979552:0:261368:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572471.979555:0:261368:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572471.979558:0:261368:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572471.979566:0:261368:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572471.979567:0:261368:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.979570:0:261368:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572471.979573:0:261368:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572471.979576:0:261368:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572471.979579:0:261368:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572471.979580:0:261368:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.979582:0:261368:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.979585:0:261368:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.979587:0:261368:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.979590:0:261368:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.979593:0:261368:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.979594:0:261368:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572471.979595:0:261368:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572471.979597:0:261368:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572471.979599:0:261368:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20bbbf8 00010000:00000001:0.0:1788572471.979601:0:261368:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572471.979602:0:261368:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572471.979603:0:261368:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572471.979605:0:261368:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.979607:0:261368:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572471.979610:0:261368:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572471.979612:0:261368:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 00010000:00000001:0.0:1788572471.979618:0:261368:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.979619:0:261368:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.979620:0:261368:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572471.979622:0:261368:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572471.979625:0:261368:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 196 lru_type: 1 00010000:00000001:0.0:1788572471.979632:0:261368:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572471.979657:0:261368:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572471.979660:0:261368:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572471.979665:0:261368:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.979666:0:261368:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572471.979668:0:261368:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.979671:0:261368:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572471.979673:0:261368:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.979676:0:261368:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.979679:0:261368:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572471.979681:0:261368:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572471.979683:0:261368:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572471.979683:0:261368:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572471.979687:0:261368:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572471.979690:0:261368:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572471.979692:0:261368:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572471.979696:0:261368:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572471.979700:0:261368:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572471.979702:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbb88 00000080:00002000:0.0:1788572471.979703:0:261368:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 48 00000080:00000001:0.0:1788572471.980016:0:261368:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572471.980019:0:261368:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20bbbd0 00000080:00200000:0.0:1788572471.980020:0:261368:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000800:00000200:0.0:1788572472.011858:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.011868:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572472.011875:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.011893:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.011902:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1ee0 00000400:00000010:0.0:1788572472.011904:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1ee0. 00000100:00000001:0.0:1788572472.011909:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.011915:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6bb80 x1875440809944960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.011926:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.011929:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6bb80 x1875440809944960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.011936:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.011938:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572472.022692:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.022697:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572472.022701:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.022706:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.022711:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1220 00000400:00000010:0.0:1788572472.022713:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1220. 00000100:00000001:0.0:1788572472.022717:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.022721:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6bb80 x1875440809945088/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.022730:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.022733:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440809945088/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.022739:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.022741:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572472.043388:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.043394:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572472.043399:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.043406:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.043412:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3a18 00000400:00000010:0.0:1788572472.043414:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3a18. 00000100:00000001:0.0:1788572472.043418:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.043422:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4e00 x1875440809945216/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.043433:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.043436:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4e00 x1875440809945216/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.043443:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.043444:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572472.064071:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.064087:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572472.064093:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.064098:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.064103:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf12a8 00000400:00000010:0.0:1788572472.064105:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf12a8. 00000100:00000001:0.0:1788572472.064109:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.064114:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70e00 x1875440809945344/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.064122:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.064126:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440809945344/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.064132:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.064134:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572472.096985:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.096990:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572472.096995:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.097001:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.097006:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3d48 00000400:00000010:0.0:1788572472.097008:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3d48. 00000100:00000001:0.0:1788572472.097012:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.097016:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6a00 x1875440809945472/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.097025:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.097028:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6a00 x1875440809945472/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.097036:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.097037:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.111994:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.112010:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.112014:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.112016:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.112025:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.112032:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.112040:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.112045:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.112051:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.112054:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417462a00 00000400:00000200:0.0:1788572472.112059:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.112064:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529dd1 [1] + 224 00000800:00000001:0.0:1788572472.112070:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.112075:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.112136:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.112142:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.112150:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.112159:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809945600/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.112170:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70e00 x1875440809945600/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.112189:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.112198:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.112201:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.134307:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.134314:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572472.134319:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.134325:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.134332:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f35d8 00000400:00000010:0.0:1788572472.134335:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f35d8. 00000100:00000001:0.0:1788572472.134340:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.134345:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809945728/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.134356:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.134359:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5f80 x1875440809945728/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.134366:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.134368:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.154691:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.154708:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.154712:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.154715:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.154724:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.154734:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.154744:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.154749:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.154754:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.154759:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417462a80 00000400:00000200:0.0:1788572472.154764:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.154771:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529dd9 [1] + 224 00000800:00000001:0.0:1788572472.154778:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.154798:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.154802:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.154807:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.154815:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.154824:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5f80 x1875440809945728/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.154835:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c5f80 x1875440809945728/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.154857:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.154865:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.154869:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.164871:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.164874:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572472.164978:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.164982:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572472.164988:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.164993:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.164999:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788572472.165001:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788572472.165005:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.165009:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809945856/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.165019:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.165023:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440809945856/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.165029:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.165031:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.167749:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.167766:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.167770:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.167772:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.167780:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.167788:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.167796:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.167801:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.167806:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.167810:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417462b00 00000400:00000200:0.0:1788572472.167814:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.167829:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529de9 [1] + 224 00000800:00000001:0.0:1788572472.167836:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.167843:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.167846:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.167851:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.167858:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.167865:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5f80 x1875440809945856/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.167875:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c5f80 x1875440809945856/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.167904:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.167912:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.167915:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.169655:0:261376:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572472.169659:0:261376:0:(cl_object.c:143:cl_object_top()) ffff9c371f02ad00 -> ffff9c371f02ad00 00000020:00000001:0.0:1788572472.169662:0:261376:0:(cl_object.c:143:cl_object_top()) ffff9c371f02ad00 -> ffff9c371f02ad00 00000020:00000001:0.0:1788572472.169664:0:261376:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572472.169668:0:261376:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572472.169672:0:261376:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.169674:0:261376:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.169676:0:261376:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.169677:0:261376:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.169679:0:261376:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdc8:11031 on OST[0]: s=0 (1) m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572472.169683:0:261376:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.169686:0:261376:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572466 a=1788572466 c=1788572466 b=0 00020000:00000001:0.0:1788572472.169688:0:261376:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.169690:0:261376:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.169691:0:261376:0:(cl_object.c:143:cl_object_top()) ffff9c371f02ad00 -> ffff9c371f02ad00 00000080:00200000:0.0:1788572472.169694:0:261376:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdc8:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572472.169696:0:261376:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572472.169698:0:261376:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572472.169700:0:261376:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572472.169703:0:261376:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.169713:0:261376:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572472.169718:0:261376:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572472.169721:0:261376:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.169723:0:261376:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.169725:0:261376:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.169727:0:261376:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192a430 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572472.169731:0:261376:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572472.169732:0:261376:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572472.169734:0:261376:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.169736:0:261376:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.169737:0:261376:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572472.169739:0:261376:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.169740:0:261376:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.169741:0:261376:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.169743:0:261376:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.169744:0:261376:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572472.169747:0:261376:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572472.169748:0:261376:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.169750:0:261376:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192a430 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572472.169753:0:261376:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313b300. 00000020:00000001:0.0:1788572472.169757:0:261376:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572472.169758:0:261376:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192a400. 00020000:00000001:0.0:1788572472.169761:0:261376:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.169762:0:261376:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.169763:0:261376:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572472.169765:0:261376:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.169768:0:261376:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572472.169771:0:261376:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572472.169773:0:261376:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572472.169775:0:261376:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572472.169776:0:261376:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.169778:0:261376:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572472.169781:0:261376:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572472.169783:0:261376:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572472.169786:0:261376:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572472.169788:0:261376:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdc8:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572472.169792:0:261376:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.169794:0:261376:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.169795:0:261376:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572472.169797:0:261376:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572472.169798:0:261376:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.169800:0:261376:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572472.169801:0:261376:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.169810:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3d08 00000080:00000008:0.0:1788572472.169813:0:261376:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-052:[0x200002b17:0xdc8:0x0](ffff9c3708765348), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.169829:0:261376:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.169831:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3da0 00000080:00002000:0.0:1788572472.169832:0:261376:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-052 (ffff9c37061ccaf8, parent ffff9c3719d93af8, inode ffff9c3708765348) hashed, 00000080:00000001:0.0:1788572472.169835:0:261376:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.169868:0:261376:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.169871:0:261376:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.169873:0:261376:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.169875:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3bc0 00000080:00200000:0.0:1788572472.169877:0:261376:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.169892:0:261376:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572472.169899:0:261376:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.169903:0:261376:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.169906:0:261376:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.169909:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3ae0 00800000:00000002:0.0:1788572472.169911:0:261376:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.169914:0:261376:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.169917:0:261376:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.169920:0:261376:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.169922:0:261376:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.169924:0:261376:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.169926:0:261376:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.169928:0:261376:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.169931:0:261376:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.169935:0:261376:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.169940:0:261376:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.169942:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c39b0 00000002:00010000:0.0:1788572472.169945:0:261376:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.169948:0:261376:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.169950:0:261376:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.169952:0:261376:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.169960:0:261376:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.169962:0:261376:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.169967:0:261376:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.169970:0:261376:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.169974:0:261376:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 203 lru_type: 1 02000000:00000001:0.0:1788572472.169983:0:261376:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.169988:0:261376:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.169991:0:261376:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 203 lru_type: 1 00000002:00000001:0.0:1788572472.169998:0:261376:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.170000:0:261376:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.170002:0:261376:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.170003:0:261376:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.170005:0:261376:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.170007:0:261376:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.170010:0:261376:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.170012:0:261376:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.170015:0:261376:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6000. 00000080:00000001:0.0:1788572472.170023:0:261376:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.170024:0:261376:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.170027:0:261376:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.170030:0:261376:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.170033:0:261376:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.170035:0:261376:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.170036:0:261376:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.170038:0:261376:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.170040:0:261376:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.170043:0:261376:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.170045:0:261376:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.170047:0:261376:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.170049:0:261376:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.170050:0:261376:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.170051:0:261376:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.170054:0:261376:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20c3bf8 00010000:00000001:0.0:1788572472.170055:0:261376:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.170056:0:261376:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.170057:0:261376:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.170059:0:261376:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.170060:0:261376:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.170062:0:261376:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.170064:0:261376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 203 lru_type: 1 00010000:00000001:0.0:1788572472.170070:0:261376:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.170070:0:261376:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.170072:0:261376:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.170073:0:261376:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.170087:0:261376:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00010000:00000001:0.0:1788572472.170094:0:261376:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.170096:0:261376:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.170097:0:261376:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.170100:0:261376:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.170101:0:261376:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.170213:0:261376:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.170217:0:261376:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.170218:0:261376:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.170220:0:261376:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.170223:0:261376:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.170225:0:261376:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.170226:0:261376:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.170227:0:261376:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.170230:0:261376:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.170232:0:261376:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.170234:0:261376:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.170238:0:261376:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.170242:0:261376:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.170244:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3b88 00000080:00002000:0.0:1788572472.170246:0:261376:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 52 00000080:00000001:0.0:1788572472.170248:0:261376:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.170251:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3bd0 00000080:00200000:0.0:1788572472.170252:0:261376:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572472.170256:0:261376:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.170260:0:261376:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.170262:0:261376:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.170264:0:261376:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.170266:0:261376:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.170268:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3be8 00000080:00002000:0.0:1788572472.170269:0:261376:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-052(ffff9c37061ccaf8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572472.170271:0:261376:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.170273:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3c30 00000080:00200000:0.0:1788572472.170274:0:261376:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-052, flags=9 00000080:00000001:0.0:1788572472.170285:0:261376:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572472.170289:0:261376:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572472.170291:0:261376:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.170292:0:261376:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20c3e40 00000080:00002000:0.0:1788572472.170293:0:261376:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-052 (ffff9c37061ccaf8, parent ffff9c3719d93af8, inode ffff9c3708765348) hashed, 00000080:00000001:0.0:1788572472.170296:0:261376:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.187531:0:261378:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.187541:0:261378:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.187543:0:261378:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.187547:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb20 00000080:00200000:0.0:1788572472.187552:0:261378:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.187561:0:261378:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572472.187567:0:261378:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.187571:0:261378:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.187576:0:261378:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.187578:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba40 00800000:00000002:0.0:1788572472.187581:0:261378:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.187585:0:261378:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.187587:0:261378:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.187590:0:261378:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.187592:0:261378:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.187594:0:261378:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.187597:0:261378:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.187599:0:261378:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.187601:0:261378:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.187605:0:261378:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.187610:0:261378:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.187612:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b910 00000002:00010000:0.0:1788572472.187614:0:261378:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.187617:0:261378:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.187619:0:261378:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.187622:0:261378:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.187629:0:261378:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.187632:0:261378:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.187668:0:261378:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.187672:0:261378:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.187677:0:261378:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 02000000:00000001:0.0:1788572472.187688:0:261378:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.187692:0:261378:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.187695:0:261378:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00000002:00000001:0.0:1788572472.187702:0:261378:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.187712:0:261378:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.187714:0:261378:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.187716:0:261378:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.187718:0:261378:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.187720:0:261378:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.187723:0:261378:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.187726:0:261378:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.187729:0:261378:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572472.187734:0:261378:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.187736:0:261378:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.187739:0:261378:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.187742:0:261378:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.187745:0:261378:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.187747:0:261378:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.187749:0:261378:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.187751:0:261378:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.187753:0:261378:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.187756:0:261378:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.187758:0:261378:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.187761:0:261378:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.187763:0:261378:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.187764:0:261378:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.187766:0:261378:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.187768:0:261378:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bb58 00010000:00000001:0.0:1788572472.187770:0:261378:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.187771:0:261378:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.187773:0:261378:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.187774:0:261378:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.187776:0:261378:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.187778:0:261378:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.187781:0:261378:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00010000:00000001:0.0:1788572472.187787:0:261378:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.187789:0:261378:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.187791:0:261378:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.187792:0:261378:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.187795:0:261378:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 205 lru_type: 1 00010000:00000001:0.0:1788572472.187802:0:261378:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.187805:0:261378:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.187807:0:261378:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.187811:0:261378:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.187812:0:261378:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.187814:0:261378:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.190865:0:261378:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.190874:0:261378:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.190880:0:261378:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.190895:0:261378:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.190899:0:261378:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.190901:0:261378:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.190904:0:261378:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.190909:0:261378:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.190910:0:261378:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.190915:0:261378:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.190924:0:261378:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.190930:0:261378:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.190934:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00002000:0.0:1788572472.190936:0:261378:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 52 00000080:00000001:0.0:1788572472.190940:0:261378:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.190942:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb30 00000080:00200000:0.0:1788572472.190944:0:261378:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572472.190951:0:261378:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.190957:0:261378:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.190958:0:261378:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.190961:0:261378:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572472.190977:0:261378:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f920. 00000020:00000001:0.0:1788572472.190982:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00200000:0.0:1788572472.190984:0:261378:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-053, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572472.190988:0:261378:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572472.190991:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b998 00000080:00200000:0.0:1788572472.190993:0:261378:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-053, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572472.191001:0:261378:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572472.191007:0:261378:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.191012:0:261378:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.191016:0:261378:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.191018:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b8b8 00800000:00000002:0.0:1788572472.191020:0:261378:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-053' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.191024:0:261378:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.191028:0:261378:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.191030:0:261378:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.191032:0:261378:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.191035:0:261378:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.191037:0:261378:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.191039:0:261378:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.191041:0:261378:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.191045:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b850 00800000:00000002:0.0:1788572472.191046:0:261378:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-053' -> mds #0 00000002:00000001:0.0:1788572472.191051:0:261378:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.191053:0:261378:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b788 00000002:00010000:0.0:1788572472.191055:0:261378:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-053,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572472.191060:0:261378:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572472.191062:0:261378:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572472.191067:0:261378:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572472.191071:0:261378:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572472.191239:0:261378:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.191244:0:261378:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.191250:0:261378:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.191252:0:261378:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.191253:0:261378:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.191256:0:261378:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.191259:0:261378:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.191264:0:261378:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.191272:0:261378:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572472.191279:0:261378:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.191284:0:261378:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.191286:0:261378:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.191287:0:261378:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.191288:0:261378:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.191290:0:261378:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.191297:0:261378:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00010000:00000001:0.0:1788572472.191301:0:261378:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.191303:0:261378:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572472.191309:0:261378:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572472.191312:0:261378:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.191315:0:261378:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.191319:0:261378:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fc200. 00000020:00000001:0.0:1788572472.191323:0:261378:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.191326:0:261378:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fc200 with handle 0x14415860931685cd to hash 00000020:00000001:0.0:1788572472.191328:0:261378:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.191329:0:261378:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790930944 : -109714918620672 : ffff9c37013fc200) 00010000:00000001:0.0:1788572472.191333:0:261378:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790930944 : -109714918620672 : ffff9c37013fc200) 00010000:00000001:0.0:1788572472.191336:0:261378:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.191339:0:261378:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.191343:0:261378:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc200/0x14415860931685cd lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261378 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.191352:0:261378:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc200/0x14415860931685cd lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261378 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.191363:0:261378:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc200/0x14415860931685cd lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261378 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572472.191368:0:261378:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572472.191370:0:261378:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572472.191376:0:261378:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572472.191379:0:261378:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572472.191381:0:261378:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.191382:0:261378:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572472.191386:0:261378:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.191391:0:261378:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.191393:0:261378:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572472.191394:0:261378:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572472.191405:0:261378:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809945984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.191413:0:261378:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572472.191415:0:261378:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.191419:0:261378:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572472.191422:0:261378:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572472.191424:0:261378:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572472.191427:0:261378:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261378:1875440809945984:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.191431:0:261378:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572472.191437:0:261378:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572472.191440:0:261378:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.191443:0:261378:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572472.191448:0:261378:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb800. 02000000:00000001:0.0:1788572472.191450:0:261378:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572472.191458:0:261378:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572472.191462:0:261378:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c89a0. 00000400:00000010:0.0:1788572472.191469:0:261378:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572472.191475:0:261378:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809945984, portal 10 00000100:00000001:0.0:1788572472.191477:0:261378:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.191479:0:261378:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572472.191484:0:261378:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6d80 x1875440809945984/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.191491:0:261378:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572472.191493:0:261378:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572472.191496:0:261378:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572472.191499:0:261378:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809945984, offset 0 00000400:00000200:0.0:1788572472.191505:0:261378:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572472.191513:0:261378:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.191521:0:261378:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572472.191527:0:261378:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572472.191532:0:261378:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572472.191535:0:261378:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572472.191538:0:261378:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572472.191542:0:261378:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539770:539770:256:4294967295] 192.168.202.148@tcp LPNI seq info [539735:539735:8:4294967295] 00000400:00000200:0.0:1788572472.191547:0:261378:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.191556:0:261378:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572472.191562:0:261378:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572472.191566:0:261378:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572472.191573:0:261378:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572472.191581:0:261378:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572472.191585:0:261378:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0200 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572472.191696:0:261378:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.191701:0:261378:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14988 usecs (15 jiffies) 00000400:00000200:0.0:1788572472.191705:0:261378:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572472.191708:0:261378:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307611207 00000100:00000001:0.0:1788572472.191711:0:261378:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.191714:0:261378:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307611207 00000100:00000001:0.0:1788572472.191716:0:261378:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.191719:0:261378:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.191721:0:261378:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.191723:0:261378:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572472.191725:0:261378:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572472.191728:0:261378:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572472.191730:0:261378:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572472.191733:0:261378:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572472.242042:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.242048:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572472.242053:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.242058:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.242064:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3cc0 00000400:00000010:0.0:1788572472.242067:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3cc0. 00000100:00000001:0.0:1788572472.242071:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.242075:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809946240/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.242084:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.242087:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5f80 x1875440809946240/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.242094:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.242096:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.365513:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.365530:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.365534:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.365537:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.365547:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.365556:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.365564:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.365569:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.365574:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.365578:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417462e80 00000400:00000200:0.0:1788572472.365584:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.365590:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529e39 [1] + 224 00000800:00000001:0.0:1788572472.365597:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.365604:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.365607:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.365612:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.365618:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.365627:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809946752/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.365668:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec72680 x1875440809946752/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.365683:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.365691:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.365695:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.368509:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.368515:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572472.368520:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.368526:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.368531:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1770 00000400:00000010:0.0:1788572472.368533:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1770. 00000100:00000001:0.0:1788572472.368537:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.368541:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809946880/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.368551:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.368554:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440809946880/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.368561:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.368562:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.374847:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.374865:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.374870:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.374872:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.374881:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.374899:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.374910:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.374915:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.374920:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.374924:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417462f00 00000400:00000200:0.0:1788572472.374929:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.374935:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529e41 [1] + 224 00000800:00000001:0.0:1788572472.374942:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.374950:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.374953:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.374958:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.374965:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.374974:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809946880/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.374986:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec72680 x1875440809946880/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.375005:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.375014:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.375017:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.416428:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.416447:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.416451:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.416454:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.416462:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.416471:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.416480:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.416485:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.416489:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.416494:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417462f80 00000400:00000200:0.0:1788572472.416499:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.416506:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529e49 [1] + 224 00000800:00000001:0.0:1788572472.416513:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.416521:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.416524:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.416528:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.416535:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.416544:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809947008/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.416555:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec71f80 x1875440809947008/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.416579:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.416586:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.416589:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.418750:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.418755:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572472.418760:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.418765:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.418769:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1c38 00000400:00000010:0.0:1788572472.418772:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1c38. 00000100:00000001:0.0:1788572472.418775:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.418780:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809947136/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.418789:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.418792:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809947136/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.418797:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.418799:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.428066:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.428089:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.428094:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.428097:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.428107:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.428116:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.428128:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.428133:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.428139:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.428144:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463000 00000400:00000200:0.0:1788572472.428150:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.428158:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529e51 [1] + 224 00000800:00000001:0.0:1788572472.428167:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.428181:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.428186:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.428192:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.428201:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.428211:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809947136/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.428230:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809947136/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.428259:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.428267:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.428271:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.467361:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.467378:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.467383:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.467386:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.467395:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.467403:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.467412:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.467416:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.467421:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.467425:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463080 00000400:00000200:0.0:1788572472.467430:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.467435:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529e59 [1] + 224 00000800:00000001:0.0:1788572472.467442:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.467448:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.467450:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.467455:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.467461:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.467469:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4380 x1875440809947264/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.467480:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4380 x1875440809947264/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.467502:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.467509:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.467512:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.469536:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.469539:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572472.477299:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.477320:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.477324:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.477327:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.477337:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.477344:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.477353:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.477358:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.477364:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.477368:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463100 00000400:00000200:0.0:1788572472.477373:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.477379:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529e69 [1] + 224 00000800:00000001:0.0:1788572472.477386:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.477393:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.477397:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.477401:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.477409:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.477417:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4380 x1875440809947392/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.477428:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c4380 x1875440809947392/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.477451:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.477461:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.477464:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.507805:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.507812:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572472.507824:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.507830:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.507837:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1990 00000400:00000010:0.0:1788572472.507839:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1990. 00000100:00000001:0.0:1788572472.507844:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.507849:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68000 x1875440809947520/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.507859:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.507863:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68000 x1875440809947520/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.507870:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.507872:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000080:00000001:0.0:1788572472.613493:0:261394:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.613499:0:261394:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.613502:0:261394:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.613505:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3b20 00000080:00200000:0.0:1788572472.613511:0:261394:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.613519:0:261394:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572472.613526:0:261394:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.613530:0:261394:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.613536:0:261394:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.613538:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3a40 00800000:00000002:0.0:1788572472.613540:0:261394:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.613544:0:261394:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.613547:0:261394:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.613550:0:261394:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.613553:0:261394:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.613555:0:261394:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.613557:0:261394:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.613559:0:261394:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.613561:0:261394:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.613565:0:261394:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.613570:0:261394:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.613572:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3910 00000002:00010000:0.0:1788572472.613574:0:261394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.613577:0:261394:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.613579:0:261394:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.613582:0:261394:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.613590:0:261394:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.613593:0:261394:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.613597:0:261394:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.613602:0:261394:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.613607:0:261394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 220 lru_type: 1 02000000:00000001:0.0:1788572472.613618:0:261394:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.613623:0:261394:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.613626:0:261394:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 220 lru_type: 1 00000002:00000001:0.0:1788572472.615212:0:261394:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.615219:0:261394:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.615222:0:261394:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.615225:0:261394:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.615227:0:261394:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.615231:0:261394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.615234:0:261394:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.615238:0:261394:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.615242:0:261394:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572472.615251:0:261394:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.615253:0:261394:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.615256:0:261394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.615260:0:261394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.615264:0:261394:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.615267:0:261394:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.615269:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.615272:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.615275:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.615278:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.615281:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.615284:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.615286:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.615287:0:261394:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.615288:0:261394:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.615291:0:261394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20e3b58 00010000:00000001:0.0:1788572472.615294:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.615295:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.615297:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.615299:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.615301:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.615303:0:261394:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.615307:0:261394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 220 lru_type: 1 00010000:00000001:0.0:1788572472.615315:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.615316:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.615318:0:261394:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.615320:0:261394:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.615323:0:261394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 221 lru_type: 1 00010000:00000001:0.0:1788572472.615330:0:261394:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.615333:0:261394:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.615335:0:261394:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.615340:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.615341:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.615343:0:261394:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615346:0:261394:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.615348:0:261394:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615350:0:261394:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615353:0:261394:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615355:0:261394:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.615357:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.615357:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.615360:0:261394:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.615362:0:261394:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.615365:0:261394:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.615370:0:261394:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.615374:0:261394:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.615376:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3ae8 00000080:00002000:0.0:1788572472.615379:0:261394:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 60 00000080:00000001:0.0:1788572472.615382:0:261394:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.615383:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3b30 00000080:00200000:0.0:1788572472.615385:0:261394:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572472.615391:0:261394:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.615396:0:261394:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.615398:0:261394:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.615400:0:261394:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572472.615416:0:261394:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f5e0. 00000020:00000001:0.0:1788572472.615419:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3ae8 00000080:00200000:0.0:1788572472.615421:0:261394:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-061, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572472.615424:0:261394:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572472.615426:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3998 00000080:00200000:0.0:1788572472.615428:0:261394:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-061, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572472.615435:0:261394:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572472.615438:0:261394:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.615442:0:261394:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.615444:0:261394:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.615446:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e38b8 00800000:00000002:0.0:1788572472.615447:0:261394:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-061' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.615450:0:261394:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.615454:0:261394:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.615456:0:261394:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.615458:0:261394:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.615460:0:261394:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.615462:0:261394:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.615464:0:261394:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.615466:0:261394:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.615469:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3850 00800000:00000002:0.0:1788572472.615470:0:261394:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-061' -> mds #0 00000002:00000001:0.0:1788572472.615474:0:261394:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.615475:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3788 00000002:00010000:0.0:1788572472.615477:0:261394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-061,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572472.615481:0:261394:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572472.615484:0:261394:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572472.615489:0:261394:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572472.615493:0:261394:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572472.615498:0:261394:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.615501:0:261394:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.615504:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.615505:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.615507:0:261394:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.615509:0:261394:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.615512:0:261394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.615515:0:261394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.615522:0:261394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572472.615530:0:261394:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615535:0:261394:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.615537:0:261394:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615538:0:261394:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.615540:0:261394:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.615542:0:261394:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.615547:0:261394:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00010000:00000001:0.0:1788572472.615563:0:261394:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.615566:0:261394:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572472.615572:0:261394:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572472.615574:0:261394:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.615578:0:261394:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.615581:0:261394:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fda00. 00000020:00000001:0.0:1788572472.615584:0:261394:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.615587:0:261394:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fda00 with handle 0x144158609316863d to hash 00000020:00000001:0.0:1788572472.615590:0:261394:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.615591:0:261394:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.615595:0:261394:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.615599:0:261394:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.615601:0:261394:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.615606:0:261394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261394 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.615618:0:261394:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261394 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.615628:0:261394:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261394 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572472.617062:0:261394:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572472.617069:0:261394:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572472.617192:0:261394:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572472.617197:0:261394:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572472.617202:0:261394:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.617203:0:261394:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788572472.617208:0:261394:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.617212:0:261394:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.617214:0:261394:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572472.617216:0:261394:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572472.617227:0:261394:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.617237:0:261394:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572472.617239:0:261394:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.617243:0:261394:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572472.617247:0:261394:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572472.617249:0:261394:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572472.617253:0:261394:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261394:1875440809948032:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.617258:0:261394:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572472.617264:0:261394:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572472.617266:0:261394:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.617268:0:261394:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572472.617275:0:261394:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572472.617277:0:261394:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572472.617284:0:261394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572472.617288:0:261394:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8790. 00000400:00000010:0.0:1788572472.617295:0:261394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572472.617301:0:261394:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809948032, portal 10 00000100:00000001:0.0:1788572472.617303:0:261394:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.617304:0:261394:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000040:0.0:1788572472.617312:0:261394:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.617319:0:261394:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572472.617322:0:261394:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572472.617324:0:261394:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572472.617327:0:261394:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809948032, offset 0 00000400:00000200:0.0:1788572472.617333:0:261394:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572472.617341:0:261394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.617349:0:261394:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572472.617355:0:261394:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572472.617358:0:261394:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572472.617362:0:261394:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572472.617365:0:261394:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572472.617369:0:261394:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539786:539786:256:4294967295] 192.168.202.148@tcp LPNI seq info [539751:539751:8:4294967295] 00000400:00000200:0.0:1788572472.617373:0:261394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.617382:0:261394:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572472.617388:0:261394:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572472.617393:0:261394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572472.617400:0:261394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572472.617408:0:261394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572472.617413:0:261394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572472.617489:0:261394:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.617494:0:261394:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14993 usecs (15 jiffies) 00000400:00000200:0.0:1788572472.617498:0:261394:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572472.617500:0:261394:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307611638 00000100:00000001:0.0:1788572472.617503:0:261394:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.617505:0:261394:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307611638 00000100:00000001:0.0:1788572472.617508:0:261394:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.617510:0:261394:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.617512:0:261394:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.617514:0:261394:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572472.617516:0:261394:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572472.617518:0:261394:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572472.617521:0:261394:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572472.617524:0:261394:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.627240:0:261394:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572472.627252:0:261394:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572472.627254:0:261394:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572472.627258:0:261394:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8790. 00000400:00000200:0.0:1788572472.627264:0:261394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31990 00000400:00000010:0.0:1788572472.627266:0:261394:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31990. 00000100:00000001:0.0:1788572472.627270:0:261394:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.627276:0:261394:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572472.627286:0:261394:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.627293:0:261394:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572472.627296:0:261394:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.627298:0:261394:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572472.627302:0:261394:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572472.627305:0:261394:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572472.627308:0:261394:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.627311:0:261394:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572472.627321:0:261394:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=36 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572472.627326:0:261394:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=68 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572472.627328:0:261394:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572472.627331:0:261394:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.627334:0:261394:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572472.627338:0:261394:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.627342:0:261394:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572472.627345:0:261394:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572472.627347:0:261394:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572472.627350:0:261394:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.627355:0:261394:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.627363:0:261394:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.627365:0:261394:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572472.627368:0:261394:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572472.627372:0:261394:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572472.627380:0:261394:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261394:1875440809948032:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.627385:0:261394:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.627387:0:261394:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.627390:0:261394:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572472.627392:0:261394:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.627395:0:261394:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.627401:0:261394:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572472.627403:0:261394:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572472.627407:0:261394:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572472.627408:0:261394:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.627411:0:261394:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572472.627414:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627416:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627419:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=4 00000020:00000001:0.0:1788572472.627422:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627424:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000040:0.0:1788572472.627431:0:261394:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37013fda00, remote cookie: 0xd9894672d99a26c0, flags: 0x1001 00010000:00010000:0.0:1788572472.627436:0:261394:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572472.627450:0:261394:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdd1:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572472.627453:0:261394:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572472.627459:0:261394:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514100 count: 2 00010000:00000001:0.0:1788572472.627461:0:261394:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.627464:0:261394:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.627467:0:261394:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.627475:0:261394:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627480:0:261394:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572472.627484:0:261394:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572472.627486:0:261394:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572472.627487:0:261394:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572472.627489:0:261394:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572472.627491:0:261394:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdd1:0x0].0x0 (ffff9c372f514100) refcount = 2 00010000:00000040:0.0:1788572472.627494:0:261394:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572472.627496:0:261394:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c371f119c00/0x144158609316801d lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a143d expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572472.627503:0:261394:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627508:0:261394:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572472.627511:0:261394:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572472.627513:0:261394:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.627515:0:261394:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572472.627517:0:261394:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.627520:0:261394:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627526:0:261394:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572472.627528:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627530:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.627531:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627532:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.627533:0:261394:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.627537:0:261394:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572472.627539:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627540:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627541:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627543:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627546:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627548:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627549:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627550:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=4 00000020:00000001:0.0:1788572472.627552:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627554:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627557:0:261394:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.627560:0:261394:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.627563:0:261394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627570:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627571:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.627572:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627573:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627574:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=5 00000020:00000001:0.0:1788572472.627576:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627578:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627580:0:261394:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.627582:0:261394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627589:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627590:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572472.627592:0:261394:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627599:0:261394:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.627600:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627601:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.627601:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627602:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572472.627607:0:261394:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572472.627616:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627618:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627619:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627621:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627623:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00000002:00010000:0.0:1788572472.627626:0:261394:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572472.627670:0:261394:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d091a80. 00010000:00000001:0.0:1788572472.627673:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627674:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.627676:0:261394:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.627677:0:261394:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.627680:0:261394:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572472.627682:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627683:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627685:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627686:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627688:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00000002:00010000:0.0:1788572472.627692:0:261394:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.627698:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627699:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.627701:0:261394:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572472.627702:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627703:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627714:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627716:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627718:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000040:0.0:1788572472.627721:0:261394:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514100 count: 3 00010000:00000001:0.0:1788572472.627723:0:261394:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572472.627727:0:261394:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3537 (0 0) 00010000:00000001:0.0:1788572472.627729:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627730:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.627731:0:261394:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572472.627734:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3710 00000002:00002000:0.0:1788572472.627736:0:261394:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-061 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572472.627740:0:261394:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.627743:0:261394:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.627746:0:261394:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572472.627749:0:261394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572472.627752:0:261394:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.627755:0:261394:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.627758:0:261394:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572472.627761:0:261394:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc20e3b20 it_disposition b 00000080:00000001:0.0:1788572472.627764:0:261394:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572472.627769:0:261394:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572472.627772:0:261394:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572472.627774:0:261394:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329105 : 144115373145329105 : 200002b17000dd1) 00000080:00000001:0.0:1788572472.627777:0:261394:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572472.627784:0:261394:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.627788:0:261394:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.627790:0:261394:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.627793:0:261394:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572472.627798:0:261394:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572472.627800:0:261394:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572472.627806:0:261394:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.627810:0:261394:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.627816:0:261394:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572472.627818:0:261394:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572472.627821:0:261394:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdd1:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572472.627835:0:261394:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572472.627839:0:261394:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20999 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572472.627842:0:261394:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4b0f8) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.627846:0:261394:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572472.627848:0:261394:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4b0f8) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572472.627852:0:261394:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.627855:0:261394:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.627858:0:261394:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572472.627861:0:261394:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd1:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572472.627863:0:261394:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.627865:0:261394:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572472.627867:0:261394:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572472.627871:0:261394:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329105 : 144115373145329105 : 200002b17000dd1) 00000080:00000001:0.0:1788572472.627873:0:261394:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572472.627878:0:261394:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdd1:0x0](ffff9c3708761148): rc = 0 00000080:00000001:0.0:1788572472.627881:0:261394:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911930696 : -109714797620920 : ffff9c3708761148) 00010000:00000001:0.0:1788572472.627883:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627893:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627895:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627897:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.627900:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00000080:00000001:0.0:1788572472.627903:0:261394:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572472.627904:0:261394:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.627905:0:261394:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.627907:0:261394:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.627909:0:261394:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572472.627910:0:261394:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572472.627911:0:261394:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572472.627914:0:261394:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572472.627916:0:261394:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.627919:0:261394:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572472.627920:0:261394:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572472.627923:0:261394:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdd1:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572472.627927:0:261394:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572472.627930:0:261394:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:20999 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572472.627933:0:261394:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4b0f8) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.627935:0:261394:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572472.627936:0:261394:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4b0f8) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572472.627939:0:261394:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.627941:0:261394:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572472.627943:0:261394:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572472.627945:0:261394:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd1:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572472.627946:0:261394:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.627948:0:261394:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.627951:0:261394:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572472.627953:0:261394:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b0f8) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000002:0.0:1788572472.627956:0:261394:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c372dbfd2a0 addref 2/0 by ffff9c372d1c4f80. 00020000:00000002:0.0:1788572472.627959:0:261394:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b0f8) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.627962:0:261394:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572472.627963:0:261394:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572472.627965:0:261394:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572472.627967:0:261394:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572472.627969:0:261394:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572472.627971:0:261394:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdd1:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572472.627973:0:261394:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572472.627975:0:261394:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.627977:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.627978:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.627979:0:261394:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572472.627982:0:261394:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572472.627984:0:261394:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.627987:0:261394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd1:0x0](ffff9c3708761148) for lock 0x144158609316863d 00800000:00000001:0.0:1788572472.627991:0:261394:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.627993:0:261394:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.627994:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.627995:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.627997:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.627998:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628001:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628003:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628004:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.628005:0:261394:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.628007:0:261394:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572472.628015:0:261394:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f95e8 inode ffff9c3708761148 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572472.628018:0:261394:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572472.628021:0:261394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd1:0x0](ffff9c3708761148) 00000080:00010000:0.0:1788572472.628025:0:261394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316863d from it ffffb95bc20e3b20 00010000:00000001:0.0:1788572472.628026:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.628028:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.628029:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.628031:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628033:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628035:0:261394:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.628037:0:261394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.628044:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628045:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.628046:0:261394:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.628048:0:261394:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.628051:0:261394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.628062:0:261394:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.628065:0:261394:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.628066:0:261394:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.628070:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628071:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.628073:0:261394:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628076:0:261394:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.628078:0:261394:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628144:0:261394:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628147:0:261394:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628150:0:261394:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.628151:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628152:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.628154:0:261394:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572472.628158:0:261394:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000100:00000001:0.0:1788572472.628165:0:261394:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.628170:0:261394:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440809948032/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.628177:0:261394:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572472.628180:0:261394:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572472.628183:0:261394:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572472.628186:0:261394:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572472.628188:0:261394:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572472.628190:0:261394:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572472.628192:0:261394:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572472.628195:0:261394:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572472.628198:0:261394:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572472.628200:0:261394:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572472.628201:0:261394:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788572472.628204:0:261394:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572472.628205:0:261394:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.628208:0:261394:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572472.628209:0:261394:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc20e3b20 released 00000080:00000001:0.0:1788572472.628211:0:261394:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572472.628222:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3d08 00000080:00200008:0.0:1788572472.628225:0:261394:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-061:[0x200002b17:0xdd1:0x0](ffff9c3708761148), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.628231:0:261394:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.628232:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3c80 00000080:00200000:0.0:1788572472.628234:0:261394:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdd1:0x0](ffff9c3708761148),name=f434.sanity-061 00000080:00000010:0.0:1788572472.628239:0:261394:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6e00. 00800000:00000002:0.0:1788572472.628244:0:261394:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.628249:0:261394:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.628251:0:261394:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.628253:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3ba0 00800000:00000002:0.0:1788572472.628255:0:261394:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdd1:0x0] 'f434.sanity-061' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.628258:0:261394:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.628262:0:261394:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.628264:0:261394:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.628267:0:261394:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.628269:0:261394:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.628271:0:261394:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.628273:0:261394:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.628275:0:261394:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.628277:0:261394:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.628279:0:261394:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.628280:0:261394:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.628281:0:261394:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.628283:0:261394:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.628284:0:261394:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdd1:0x0] 00800000:00000001:0.0:1788572472.628286:0:261394:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.628289:0:261394:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdd1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.628293:0:261394:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.628295:0:261394:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20e3a70 00000002:00010000:0.0:1788572472.628297:0:261394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdd1:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.628300:0:261394:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.628302:0:261394:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.628303:0:261394:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.628307:0:261394:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c372f514100 count: 3 00010000:00000001:0.0:1788572472.628309:0:261394:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.628311:0:261394:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.628315:0:261394:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.628318:0:261394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572472.628326:0:261394:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.628329:0:261394:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.628332:0:261394:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572472.628338:0:261394:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.628340:0:261394:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.628341:0:261394:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.628343:0:261394:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.628345:0:261394:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.628346:0:261394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572472.628348:0:261394:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.628350:0:261394:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572472.628352:0:261394:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6e00. 00000080:00000001:0.0:1788572472.628355:0:261394:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.628357:0:261394:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.628359:0:261394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd1:0x0](ffff9c3708761148) 00000080:00010000:0.0:1788572472.628361:0:261394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd1:0x0](ffff9c3708761148) for lock 0x144158609316863d 00800000:00000001:0.0:1788572472.628364:0:261394:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.628365:0:261394:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.628366:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.628367:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.628369:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.628371:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628373:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628375:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628376:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.628377:0:261394:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.628379:0:261394:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.628380:0:261394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316863d from it ffffb95bc20e3cb8 00010000:00000001:0.0:1788572472.628382:0:261394:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.628383:0:261394:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.628384:0:261394:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fda00 refcount=3 00000020:00000001:0.0:1788572472.628386:0:261394:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628388:0:261394:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937088 : -109714918614528 : ffff9c37013fda00) 00010000:00000001:0.0:1788572472.628390:0:261394:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.628392:0:261394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.628398:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628398:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.628399:0:261394:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.628400:0:261394:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.628403:0:261394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fda00/0x144158609316863d lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a26c0 expref: -99 pid: 261394 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572472.628408:0:261394:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.628409:0:261394:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.628410:0:261394:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.628412:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628413:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.628414:0:261394:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628416:0:261394:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.628417:0:261394:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628419:0:261394:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628421:0:261394:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628422:0:261394:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.628423:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628424:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.628426:0:261394:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.628427:0:261394:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.628430:0:261394:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572472.628432:0:261394:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.628435:0:261394:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.628437:0:261394:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.628438:0:261394:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.628441:0:261394:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.628444:0:261394:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572472.628446:0:261394:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.628447:0:261394:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.628450:0:261394:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.628455:0:261394:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c468 -> ffff9c3730f6c468 00000020:00000001:0.0:1788572472.628458:0:261394:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572472.628460:0:261394:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572472.628462:0:261394:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdd1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572472.628465:0:261394:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.628466:0:261394:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572472.628468:0:261394:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.628470:0:261394:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.628472:0:261394:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.628476:0:261394:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdd1:0x0]io ffff9c371ff50bf0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572472.628479:0:261394:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b0f8) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.628482:0:261394:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572472.628484:0:261394:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572472.628486:0:261394:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572472.628488:0:261394:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628490:0:261394:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572472.628492:0:261394:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572472.628494:0:261394:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628495:0:261394:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.628496:0:261394:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.628497:0:261394:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572472.628499:0:261394:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b0f8) owner 0000000000000000/ffff9c372d1c4f80 00000020:00000001:0.0:1788572472.628501:0:261394:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.628503:0:261394:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572472.628505:0:261394:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdd1:0x0] 00000020:00000001:0.0:1788572472.628507:0:261394:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572472.628509:0:261394:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572472.628511:0:261394:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572472.628514:0:261394:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572472.628519:0:261394:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572472.628524:0:261394:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676900. 00020000:00000002:0.0:1788572472.628526:0:261394:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572472.628529:0:261394:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572472.628531:0:261394:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.628533:0:261394:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572472.628534:0:261394:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.628536:0:261394:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572472.628538:0:261394:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572472.628540:0:261394:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c148 -> ffff9c3730f6c468 00000020:00000001:0.0:1788572472.628542:0:261394:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572472.628545:0:261394:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.628546:0:261394:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572472.628547:0:261394:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628549:0:261394:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628551:0:261394:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.628552:0:261394:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572472.628557:0:261394:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a7300. 00000008:00010000:0.0:1788572472.628560:0:261394:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572472.628562:0:261394:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.628564:0:261394:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572472.628565:0:261394:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676930, osc lock ffff9c37030a7300, flags 0x400001002 00000020:00000001:0.0:1788572472.628567:0:261394:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628569:0:261394:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628572:0:261394:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531727104 : -109714177824512 : ffff9c372d676900) 00000020:00000001:0.0:1788572472.628573:0:261394:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.628574:0:261394:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.628576:0:261394:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628577:0:261394:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628580:0:261394:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572472.628582:0:261394:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572472.628583:0:261394:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.628585:0:261394:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.628586:0:261394:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.628587:0:261394:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.628588:0:261394:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.628591:0:261394:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.628593:0:261394:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572472.628596:0:261394:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572472.628599:0:261394:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.628600:0:261394:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.628601:0:261394:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572472.628603:0:261394:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572472.628604:0:261394:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572472.628607:0:261394:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572472.628608:0:261394:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572472.628612:0:261394:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628615:0:261394:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572472.628619:0:261394:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572472.628622:0:261394:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572472.628626:0:261394:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.628630:0:261394:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.628633:0:261394:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.628661:0:261394:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.628662:0:261394:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.628665:0:261394:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.628667:0:261394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.628670:0:261394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.628676:0:261394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6e00. 00000100:00000001:0.0:1788572472.628682:0:261394:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628686:0:261394:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.628688:0:261394:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628689:0:261394:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.628690:0:261394:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628692:0:261394:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.628694:0:261394:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00010000:00000001:0.0:1788572472.628697:0:261394:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.628699:0:261394:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572472.628704:0:261394:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0c00. 00010000:00000010:0.0:1788572472.628708:0:261394:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0800. 00010000:00000001:0.0:1788572472.628711:0:261394:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.628714:0:261394:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fdc00. 00000020:00000001:0.0:1788572472.628717:0:261394:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.628720:0:261394:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fdc00 with handle 0x1441586093168644 to hash 00000020:00000001:0.0:1788572472.628722:0:261394:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.628724:0:261394:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000010:0.0:1788572472.628728:0:261394:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091800. 00010000:00000001:0.0:1788572472.628730:0:261394:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.628732:0:261394:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.628734:0:261394:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.628737:0:261394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.628747:0:261394:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.628757:0:261394:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.628762:0:261394:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.628766:0:261394:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.628773:0:261394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572472.628799:0:261394:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572472.628802:0:261394:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628804:0:261394:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.628806:0:261394:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628808:0:261394:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.628810:0:261394:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572472.628812:0:261394:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572472.628814:0:261394:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572472.628815:0:261394:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572472.628827:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572472.628831:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572472.628833:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.628839:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572472.628841:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.628843:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.628845:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572472.628847:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572472.628854:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.628863:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572472.628865:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.628869:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572472.628872:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572472.628874:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572472.628878:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809948160:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.628883:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572472.628900:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572472.628902:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.628904:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572472.628909:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572472.628911:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572472.628918:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572472.628922:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8790. 00000400:00000010:0.0:1788572472.628929:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572472.628936:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809948160, portal 4 00000100:00000001:0.0:1788572472.628938:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.628940:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000040:0.0:1788572472.628946:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.628952:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572472.628955:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572472.628958:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572472.628960:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809948160, offset 0 00000400:00000200:0.0:1788572472.628966:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572472.628974:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.628982:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572472.628987:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572472.628991:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572472.628995:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572472.628997:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572472.629001:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539787:539787:256:4294967295] 192.168.202.148@tcp LPNI seq info [539752:539752:8:4294967295] 00000400:00000200:0.0:1788572472.629006:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.629015:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572472.629023:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572472.629028:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788572472.629035:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572472.629043:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572472.629047:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0e00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572472.629057:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.629061:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 177940 usecs (178 jiffies) 00000400:00000200:0.0:1788572472.629064:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572472.629066:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307611813 00000100:00000001:0.0:1788572472.629069:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.629071:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307611813 00000100:00000001:0.0:1788572472.629073:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.629076:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.629090:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.629093:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.629096:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.651741:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.651746:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572472.651752:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572472.651754:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572472.651759:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8790. 00000400:00000200:0.0:1788572472.651763:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31990 00000400:00000010:0.0:1788572472.651766:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31990. 00000100:00000001:0.0:1788572472.651769:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.651774:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572472.651784:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.651791:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572472.651793:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.651795:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572472.651798:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572472.651800:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572472.651803:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.651806:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572472.651817:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=36 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572472.651925:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=9 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572472.651928:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572472.651930:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.651933:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572472.651936:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.651941:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572472.651942:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572472.651945:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572472.651947:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.651952:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.651959:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.651962:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572472.651964:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572472.651967:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572472.651969:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.651972:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.651974:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=4 00000020:00000001:0.0:1788572472.651976:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.651980:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.651982:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.651983:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.651985:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=5 00000020:00000001:0.0:1788572472.651986:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.651988:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.651992:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.651996:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.652000:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652018:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652019:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.652021:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572472.652023:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.652024:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.652026:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=6 00000020:00000001:0.0:1788572472.652028:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652030:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000040:0.0:1788572472.652038:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37013fdc00, remote cookie: 0xd9894672d99a26c7, flags: 0x1001 00010000:00010000:0.0:1788572472.652043:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652051:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572472.652053:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652056:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572472.652059:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572472.652064:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572472.652177:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x5207:0x0].0x0 (ffff9c37198c0c00) refcount = 1 00010000:00000040:0.0:1788572472.652181:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572472.652184:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652194:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572472.652196:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.652199:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572472.652201:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.652205:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652213:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572472.652214:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652215:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.652216:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652217:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.652219:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572472.652221:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572472.652225:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.652226:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.652227:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=5 00000020:00000001:0.0:1788572472.652229:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652231:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00000008:00000001:0.0:1788572472.652236:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572472.652240:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6cfc0 -> ffff9c3730f6c468 00000008:00010000:0.0:1788572472.652244:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572472.652251:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6cfc0 -> ffff9c3730f6c468 00000020:00000001:0.0:1788572472.652254:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572472.652256:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572472.652259:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572472.652261:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.652263:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.652265:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6cfc0 -> ffff9c3730f6c468 00000008:00000001:0.0:1788572472.652266:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572472.652269:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572472.652271:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572472.652346:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572472.652349:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652351:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.652352:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.652354:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=6 00000020:00000001:0.0:1788572472.652356:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652358:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652361:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.652364:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652371:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652372:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572472.652375:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652382:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.652383:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652384:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.652385:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652387:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.652388:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.652389:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fdc00 refcount=5 00000020:00000001:0.0:1788572472.652391:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652393:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790937600 : -109714918614016 : ffff9c37013fdc00) 00010000:00000001:0.0:1788572472.652395:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.652397:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.652403:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652404:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.652405:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.652407:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.652410:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000f8a08d86/0x1441586093168644 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a26c7 expref: -99 pid: 261394 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.652417:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.652420:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.652422:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.652425:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652426:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.652428:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652431:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.652432:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652434:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652436:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.652438:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.652439:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652440:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.652441:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.652442:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.652443:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.652448:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572472.652457:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809948160:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.652462:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.652465:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.652468:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440809948160/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.652475:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572472.652477:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572472.652480:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572472.652484:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572472.652486:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572472.652488:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572472.652490:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572472.652492:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b6e00. 02000000:00000001:0.0:1788572472.652498:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572472.652500:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572472.652502:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788572472.652505:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572472.652506:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.652510:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.652517:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572472.652518:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.652521:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.652523:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714703:0:261396:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714716:0:261396:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.714722:0:261396:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572472.714725:0:261396:0:(cl_object.c:143:cl_object_top()) ffff9c3718459918 -> ffff9c3718459918 00000020:00000001:0.0:1788572472.714726:0:261396:0:(cl_object.c:143:cl_object_top()) ffff9c3718459918 -> ffff9c3718459918 00000020:00000001:0.0:1788572472.714728:0:261396:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572472.714733:0:261396:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572472.714737:0:261396:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.714739:0:261396:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.714740:0:261396:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.714742:0:261396:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.714744:0:261396:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdd2:11031 on OST[0]: s=0 (1) m=1788572467 a=1788572467 c=1788572467 b=0 00020000:00000001:0.0:1788572472.714748:0:261396:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.714750:0:261396:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572467 a=1788572467 c=1788572467 b=0 00020000:00000001:0.0:1788572472.714753:0:261396:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714754:0:261396:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714756:0:261396:0:(cl_object.c:143:cl_object_top()) ffff9c3718459918 -> ffff9c3718459918 00000080:00200000:0.0:1788572472.714758:0:261396:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdd2:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572472.714760:0:261396:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572472.714763:0:261396:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572472.714765:0:261396:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572472.714767:0:261396:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.714768:0:261396:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572472.714771:0:261396:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572472.714774:0:261396:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.714776:0:261396:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.714777:0:261396:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.714779:0:261396:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3720e473b0 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572472.714782:0:261396:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572472.714783:0:261396:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572472.714784:0:261396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.714786:0:261396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.714788:0:261396:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572472.714789:0:261396:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.714791:0:261396:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.714792:0:261396:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.714793:0:261396:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.714794:0:261396:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572472.714797:0:261396:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572472.714798:0:261396:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.714799:0:261396:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3720e473b0 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572472.714802:0:261396:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3718db3430. 00000020:00000001:0.0:1788572472.714806:0:261396:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572472.714807:0:261396:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c3720e47380. 00020000:00000001:0.0:1788572472.714809:0:261396:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.714810:0:261396:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.714812:0:261396:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572472.714813:0:261396:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714816:0:261396:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572472.714818:0:261396:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572472.714820:0:261396:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572472.714827:0:261396:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572472.714829:0:261396:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.714831:0:261396:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572472.714834:0:261396:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572472.714836:0:261396:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572472.714838:0:261396:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572472.714841:0:261396:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdd2:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572472.714845:0:261396:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.714847:0:261396:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.714849:0:261396:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572472.714851:0:261396:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572472.714852:0:261396:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.714853:0:261396:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572472.714855:0:261396:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.714863:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbd08 00000080:00000008:0.0:1788572472.714866:0:261396:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-062:[0x200002b17:0xdd2:0x0](ffff9c3708701148), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.714871:0:261396:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.714872:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbda0 00000080:00002000:0.0:1788572472.714875:0:261396:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-062 (ffff9c37022e8798, parent ffff9c3719d93af8, inode ffff9c3708701148) hashed, 00000080:00000001:0.0:1788572472.714878:0:261396:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.714913:0:261396:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.714916:0:261396:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.714918:0:261396:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.714920:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbc0 00000080:00200000:0.0:1788572472.714922:0:261396:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.714927:0:261396:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6800. 00800000:00000002:0.0:1788572472.714932:0:261396:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.714936:0:261396:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.714940:0:261396:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.714942:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbae0 00800000:00000002:0.0:1788572472.714944:0:261396:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.714947:0:261396:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.714950:0:261396:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.714952:0:261396:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.714955:0:261396:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.714957:0:261396:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.714959:0:261396:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.714961:0:261396:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.714963:0:261396:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.714967:0:261396:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.714972:0:261396:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.714974:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb9b0 00000002:00010000:0.0:1788572472.714976:0:261396:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.714979:0:261396:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.714981:0:261396:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.714984:0:261396:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.714991:0:261396:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.714993:0:261396:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.714998:0:261396:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.715001:0:261396:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.715005:0:261396:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 223 lru_type: 1 02000000:00000001:0.0:1788572472.715014:0:261396:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.715018:0:261396:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.715022:0:261396:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 223 lru_type: 1 00000002:00000001:0.0:1788572472.715029:0:261396:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.715031:0:261396:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.715033:0:261396:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.715034:0:261396:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.715036:0:261396:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.715038:0:261396:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.715041:0:261396:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.715044:0:261396:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.715047:0:261396:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6800. 00000080:00000001:0.0:1788572472.715052:0:261396:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.715054:0:261396:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.715056:0:261396:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.715059:0:261396:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.715062:0:261396:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.715064:0:261396:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.715065:0:261396:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.715067:0:261396:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.715069:0:261396:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.715071:0:261396:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.715073:0:261396:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.715075:0:261396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.715077:0:261396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.715078:0:261396:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.715080:0:261396:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.715082:0:261396:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20fbbf8 00010000:00000001:0.0:1788572472.715084:0:261396:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.715085:0:261396:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.715087:0:261396:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.715089:0:261396:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.715090:0:261396:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.715092:0:261396:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.715095:0:261396:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 223 lru_type: 1 00010000:00000001:0.0:1788572472.715101:0:261396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.715102:0:261396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.715103:0:261396:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.715104:0:261396:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.715106:0:261396:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 224 lru_type: 1 00010000:00000001:0.0:1788572472.715112:0:261396:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.715113:0:261396:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.715115:0:261396:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.715117:0:261396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.715118:0:261396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.715119:0:261396:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.715122:0:261396:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.715123:0:261396:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.715125:0:261396:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.715127:0:261396:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.715130:0:261396:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.715131:0:261396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.715132:0:261396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.715133:0:261396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.715135:0:261396:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.715137:0:261396:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.715140:0:261396:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.715144:0:261396:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.715145:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbb88 00000080:00002000:0.0:1788572472.715147:0:261396:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 62 00000080:00000001:0.0:1788572472.715149:0:261396:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.715151:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbd0 00000080:00200000:0.0:1788572472.715152:0:261396:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572472.715157:0:261396:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.715161:0:261396:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.715162:0:261396:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.715165:0:261396:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.715167:0:261396:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.715169:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbe8 00000080:00002000:0.0:1788572472.715170:0:261396:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-062(ffff9c37022e8798) flags 0x48000e refc 0 00000080:00000001:0.0:1788572472.715172:0:261396:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.715174:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbc30 00000080:00200000:0.0:1788572472.715175:0:261396:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-062, flags=9 00000080:00000001:0.0:1788572472.715185:0:261396:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572472.715189:0:261396:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572472.715191:0:261396:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.715192:0:261396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbe40 00000080:00002000:0.0:1788572472.715194:0:261396:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-062 (ffff9c37022e8798, parent ffff9c3719d93af8, inode ffff9c3708701148) hashed, 00000080:00000001:0.0:1788572472.715197:0:261396:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572472.767520:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.767530:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572472.767537:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.767545:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.767552:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572472.767554:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572472.767559:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.767566:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4380 x1875440809948544/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.767580:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.767583:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4380 x1875440809948544/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.767590:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.767592:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572472.778146:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.778153:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572472.778157:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.778163:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.778169:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1dd0 00000400:00000010:0.0:1788572472.778171:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1dd0. 00000100:00000001:0.0:1788572472.778175:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.778179:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68000 x1875440809948672/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.778188:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.778190:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440809948672/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.778197:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.778199:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.803759:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.803764:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.803768:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.803776:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.803784:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.803795:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.803799:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.803804:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.803808:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463600 00000400:00000200:0.0:1788572472.803813:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.803820:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ed9 [1] + 224 00000800:00000001:0.0:1788572472.803828:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.803841:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.803845:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.803849:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.803857:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.803867:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68000 x1875440809948672/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.803879:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed68000 x1875440809948672/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.803910:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.803919:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.803922:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.847567:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.847574:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572472.847582:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.847590:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.847597:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1d48 00000400:00000010:0.0:1788572472.847599:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1d48. 00000100:00000001:0.0:1788572472.847604:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.847609:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809948800/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.847620:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.847624:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6b480 x1875440809948800/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.847631:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.847633:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.864263:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.864282:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.864288:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.864291:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.864302:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.864311:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.864321:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.864326:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.864332:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.864337:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463680 00000400:00000200:0.0:1788572472.864343:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.864349:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ee1 [1] + 224 00000800:00000001:0.0:1788572472.864357:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.864412:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.864416:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.864420:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.864428:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.864437:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b480 x1875440809948800/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.864446:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6b480 x1875440809948800/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.864465:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.864475:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.864478:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.867126:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.867131:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572472.867135:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.867140:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.867146:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1330 00000400:00000010:0.0:1788572472.867148:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1330. 00000100:00000001:0.0:1788572472.867152:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.867156:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809948928/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.867166:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.867170:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440809948928/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.867176:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.867177:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.870428:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.870446:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.870464:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.870467:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.870477:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.870486:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.870496:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.870501:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.870507:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.870511:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463700 00000400:00000200:0.0:1788572472.870516:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.870523:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ee9 [1] + 224 00000800:00000001:0.0:1788572472.870530:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.870537:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.870541:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.870545:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.870563:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.870573:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b480 x1875440809948928/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.870584:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6b480 x1875440809948928/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.870607:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.870615:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.870619:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.892363:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.892374:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9900. 00000400:00000200:0.0:1788572472.892383:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.892392:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.892400:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1908 00000400:00000010:0.0:1788572472.892404:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1908. 00000100:00000001:0.0:1788572472.892409:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.892417:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809949056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.892431:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.892434:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68380 x1875440809949056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.892441:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.892442:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.895932:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.895948:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.895953:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.895955:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.895965:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.895972:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.895981:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.895986:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.895992:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.895997:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463780 00000400:00000200:0.0:1788572472.896003:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.896010:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ef1 [1] + 224 00000800:00000001:0.0:1788572472.896018:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.896026:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.896030:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.896034:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.896043:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.896053:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68380 x1875440809949056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.896063:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed68380 x1875440809949056/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.896077:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.896095:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.896098:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.898521:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.898528:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572472.898533:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.898538:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.898543:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1a18 00000400:00000010:0.0:1788572472.898545:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1a18. 00000100:00000001:0.0:1788572472.898549:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.899243:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809949184/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.899255:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.899258:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440809949184/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.899264:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.899266:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.901010:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.901026:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.901030:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.901033:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.901040:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.901047:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.901055:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.901060:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.901065:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.901069:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463800 00000400:00000200:0.0:1788572472.901074:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.901087:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529ef9 [1] + 224 00000800:00000001:0.0:1788572472.901093:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.901101:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.901104:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.901108:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.901116:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.901125:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809949184/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.901135:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71500 x1875440809949184/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.901157:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.901166:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.901169:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572472.928975:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.928985:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572472.928994:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.929004:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.929013:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788572472.929015:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788572472.929021:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.929028:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4380 x1875440809949312/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.929039:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.929043:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4380 x1875440809949312/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.929050:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.929052:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.937539:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.937559:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.937562:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.937565:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.937574:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.937581:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.937590:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.937595:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.937600:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.937604:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463880 00000400:00000200:0.0:1788572472.937609:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.937615:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529f01 [1] + 224 00000800:00000001:0.0:1788572472.937622:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.937629:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.937633:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.937675:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.937684:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.937693:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4380 x1875440809949312/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.937704:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4380 x1875440809949312/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.937714:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.937723:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.937726:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572472.945068:0:261404:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.945079:0:261404:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.945151:0:261404:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.945156:0:261404:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572472.945161:0:261404:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572472.945164:0:261404:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572472.945176:0:261404:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.945182:0:261404:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.945188:0:261404:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572472.945191:0:261404:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572472.945194:0:261404:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdd6:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572472.945198:0:261404:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572472.945202:0:261404:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21116 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572472.945206:0:261404:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4bc98) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.945209:0:261404:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572472.945211:0:261404:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4bc98) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572472.945215:0:261404:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.945220:0:261404:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.945222:0:261404:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572472.945224:0:261404:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd6:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572472.945226:0:261404:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.945228:0:261404:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572472.945230:0:261404:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572472.945235:0:261404:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329110 : 144115373145329110 : 200002b17000dd6) 00000080:00000001:0.0:1788572472.945238:0:261404:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572472.945243:0:261404:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdd6:0x0](ffff9c3708765b88): rc = 0 00000080:00000001:0.0:1788572472.945246:0:261404:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911949704 : -109714797601912 : ffff9c3708765b88) 00010000:00000001:0.0:1788572472.945250:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.945253:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.945256:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ec00 refcount=3 00000020:00000001:0.0:1788572472.945258:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.945261:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00000080:00000001:0.0:1788572472.945264:0:261404:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572472.945265:0:261404:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.945266:0:261404:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.945268:0:261404:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.945270:0:261404:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572472.945272:0:261404:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572472.945273:0:261404:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572472.945275:0:261404:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572472.945278:0:261404:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.945280:0:261404:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572472.945282:0:261404:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572472.945284:0:261404:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdd6:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572472.945287:0:261404:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572472.945290:0:261404:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21116 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572472.945292:0:261404:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4bc98) owner ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.945294:0:261404:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572472.945296:0:261404:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4bc98) owner ffff9c372d1c4f80 00020000:00000010:0.0:1788572472.945298:0:261404:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.945300:0:261404:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572472.945303:0:261404:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572472.945304:0:261404:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd6:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572472.945306:0:261404:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.945308:0:261404:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.945312:0:261404:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572472.945315:0:261404:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4bc98) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000002:0.0:1788572472.945318:0:261404:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c372dbfdde0 addref 2/0 by ffff9c372d1c4f80. 00020000:00000002:0.0:1788572472.945321:0:261404:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4bc98) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.945323:0:261404:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572472.945325:0:261404:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572472.945326:0:261404:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572472.945328:0:261404:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572472.945329:0:261404:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572472.945332:0:261404:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdd6:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572472.945334:0:261404:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000080:00000001:0.0:1788572472.945336:0:261404:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.945338:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.945340:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.945342:0:261404:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572472.945346:0:261404:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572472.945349:0:261404:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.945351:0:261404:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd6:0x0](ffff9c3708765b88) for lock 0x1441586093168683 00800000:00000001:0.0:1788572472.945357:0:261404:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.945360:0:261404:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.945362:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.945363:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.945365:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ec00 refcount=3 00000020:00000001:0.0:1788572472.945366:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.945369:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.945371:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.945372:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.945373:0:261404:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.945375:0:261404:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572472.945382:0:261404:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37061cc870 inode ffff9c3708765b88 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572472.945386:0:261404:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572472.945390:0:261404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd6:0x0](ffff9c3708765b88) 00000080:00010000:0.0:1788572472.945393:0:261404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168683 from it ffffb95bc20fbb20 00010000:00000001:0.0:1788572472.945395:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.945396:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.945397:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ec00 refcount=3 00000020:00000001:0.0:1788572472.945399:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.945401:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.945403:0:261404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.945408:0:261404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.945417:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.945418:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.945420:0:261404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.945423:0:261404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.945426:0:261404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.945439:0:261404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.945442:0:261404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.945444:0:261404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.945449:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.945450:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.945452:0:261404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.945456:0:261404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.945457:0:261404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.945459:0:261404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.945462:0:261404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.945464:0:261404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.945465:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.945466:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.945468:0:261404:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572472.945471:0:261404:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711214600. 00000100:00000001:0.0:1788572472.945477:0:261404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.945488:0:261404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440809949312/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.945495:0:261404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572472.945499:0:261404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572472.945502:0:261404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c3705364000. 02000000:00000001:0.0:1788572472.945505:0:261404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572472.945508:0:261404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572472.945509:0:261404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572472.945512:0:261404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572472.945515:0:261404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371fe97800. 02000000:00000001:0.0:1788572472.945519:0:261404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572472.945521:0:261404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572472.945523:0:261404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788572472.945526:0:261404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572472.945527:0:261404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.945530:0:261404:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572472.945532:0:261404:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc20fbb20 released 00000080:00000001:0.0:1788572472.945533:0:261404:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572472.945548:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbd08 00000080:00200008:0.0:1788572472.945551:0:261404:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-066:[0x200002b17:0xdd6:0x0](ffff9c3708765b88), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.945557:0:261404:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.945558:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbc80 00000080:00200000:0.0:1788572472.945560:0:261404:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdd6:0x0](ffff9c3708765b88),name=f434.sanity-066 00000080:00000010:0.0:1788572472.945566:0:261404:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572472.945572:0:261404:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.945578:0:261404:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.945582:0:261404:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.945584:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbba0 00800000:00000002:0.0:1788572472.945586:0:261404:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdd6:0x0] 'f434.sanity-066' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.945590:0:261404:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.945594:0:261404:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.945597:0:261404:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.945599:0:261404:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.945601:0:261404:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.945603:0:261404:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.945605:0:261404:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.945608:0:261404:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.945609:0:261404:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.945611:0:261404:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.945611:0:261404:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.945612:0:261404:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.945614:0:261404:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.945615:0:261404:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdd6:0x0] 00800000:00000001:0.0:1788572472.945617:0:261404:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.945619:0:261404:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdd6:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.945623:0:261404:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.945625:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fba70 00000002:00010000:0.0:1788572472.945627:0:261404:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdd6:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.945630:0:261404:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.945631:0:261404:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.945633:0:261404:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.945660:0:261404:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c373035fb00 count: 3 00010000:00000001:0.0:1788572472.945664:0:261404:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.945668:0:261404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.945671:0:261404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.945674:0:261404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572472.945682:0:261404:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.945687:0:261404:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.945690:0:261404:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572472.945696:0:261404:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.945698:0:261404:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.945700:0:261404:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.945702:0:261404:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.946172:0:261404:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.946177:0:261404:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572472.946180:0:261404:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.946184:0:261404:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572472.946186:0:261404:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572472.946193:0:261404:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.946195:0:261404:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.946198:0:261404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd6:0x0](ffff9c3708765b88) 00000080:00010000:0.0:1788572472.946201:0:261404:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd6:0x0](ffff9c3708765b88) for lock 0x1441586093168683 00800000:00000001:0.0:1788572472.946204:0:261404:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.946206:0:261404:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.946207:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.946209:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.946211:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ec00 refcount=3 00000020:00000001:0.0:1788572472.946213:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.946215:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.946218:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.946219:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.946220:0:261404:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.946222:0:261404:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.946223:0:261404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168683 from it ffffb95bc20fbcb8 00010000:00000001:0.0:1788572472.946226:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.946227:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.946228:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372da6ec00 refcount=3 00000020:00000001:0.0:1788572472.946230:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.946232:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359535889408 : -109714173662208 : ffff9c372da6ec00) 00010000:00000001:0.0:1788572472.946233:0:261404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.946236:0:261404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.946242:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.946243:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.946244:0:261404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.946246:0:261404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.946248:0:261404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372da6ec00/0x1441586093168683 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a276f expref: -99 pid: 261404 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572472.946254:0:261404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.946255:0:261404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.946257:0:261404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.946259:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.946260:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.946261:0:261404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946263:0:261404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.946265:0:261404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946266:0:261404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946269:0:261404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946270:0:261404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.946272:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.946273:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.946275:0:261404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.946276:0:261404:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.946279:0:261404:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572472.946282:0:261404:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.946284:0:261404:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.946287:0:261404:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.946288:0:261404:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.946290:0:261404:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.946293:0:261404:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572472.946294:0:261404:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.946297:0:261404:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.946299:0:261404:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.946302:0:261404:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c2d8 -> ffff9c3730f6c2d8 00000020:00000001:0.0:1788572472.946305:0:261404:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572472.946307:0:261404:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572472.946308:0:261404:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdd6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572472.946312:0:261404:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.946313:0:261404:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572472.946315:0:261404:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.946316:0:261404:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.946318:0:261404:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.946321:0:261404:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdd6:0x0]io ffff9c371ff50bf0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572472.946324:0:261404:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4bc98) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572472.946327:0:261404:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572472.946329:0:261404:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572472.946331:0:261404:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572472.946333:0:261404:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946335:0:261404:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572472.946337:0:261404:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572472.946338:0:261404:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.946339:0:261404:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.946340:0:261404:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.946341:0:261404:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572472.946342:0:261404:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4bc98) owner 0000000000000000/ffff9c372d1c4f80 00000020:00000001:0.0:1788572472.946344:0:261404:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.946346:0:261404:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572472.946347:0:261404:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdd6:0x0] 00000020:00000001:0.0:1788572472.946350:0:261404:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572472.946351:0:261404:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572472.946354:0:261404:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572472.946356:0:261404:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572472.946360:0:261404:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572472.946365:0:261404:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676f00. 00020000:00000002:0.0:1788572472.946368:0:261404:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572472.946370:0:261404:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572472.946373:0:261404:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.946374:0:261404:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572472.946376:0:261404:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.946377:0:261404:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572472.946379:0:261404:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572472.946381:0:261404:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c530 -> ffff9c3730f6c2d8 00000020:00000001:0.0:1788572472.946383:0:261404:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572472.946386:0:261404:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.946387:0:261404:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572472.946388:0:261404:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946389:0:261404:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946391:0:261404:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.946393:0:261404:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572472.946396:0:261404:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6980. 00000008:00010000:0.0:1788572472.946399:0:261404:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572472.946401:0:261404:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.946402:0:261404:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572472.946404:0:261404:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676f30, osc lock ffff9c37030a6980, flags 0x400001002 00000020:00000001:0.0:1788572472.946406:0:261404:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946408:0:261404:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946410:0:261404:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531728640 : -109714177822976 : ffff9c372d676f00) 00000020:00000001:0.0:1788572472.946412:0:261404:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.946413:0:261404:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.946415:0:261404:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.946416:0:261404:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.946419:0:261404:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572472.946421:0:261404:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572472.946422:0:261404:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.946424:0:261404:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.946425:0:261404:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.946426:0:261404:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.946428:0:261404:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.946431:0:261404:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.946433:0:261404:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572472.946436:0:261404:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572472.946438:0:261404:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.946439:0:261404:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.946442:0:261404:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572472.946444:0:261404:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572472.946445:0:261404:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572472.946448:0:261404:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572472.946450:0:261404:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572472.946453:0:261404:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946456:0:261404:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572472.946460:0:261404:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572472.946463:0:261404:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572472.946466:0:261404:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.946471:0:261404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.946474:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.946475:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.946476:0:261404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.946479:0:261404:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.946481:0:261404:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.946484:0:261404:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.946489:0:261404:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7c00. 00000100:00000001:0.0:1788572472.946496:0:261404:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946501:0:261404:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.946503:0:261404:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946505:0:261404:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.946506:0:261404:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946508:0:261404:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.946510:0:261404:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00010000:00000001:0.0:1788572472.946513:0:261404:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.946515:0:261404:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572472.946519:0:261404:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0100. 00010000:00000010:0.0:1788572472.946522:0:261404:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0e00. 00010000:00000001:0.0:1788572472.946526:0:261404:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.946529:0:261404:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fd400. 00000020:00000001:0.0:1788572472.946533:0:261404:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.946535:0:261404:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fd400 with handle 0x144158609316868a to hash 00000020:00000001:0.0:1788572472.946537:0:261404:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.946538:0:261404:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790935552 : -109714918616064 : ffff9c37013fd400) 00010000:00000010:0.0:1788572472.946543:0:261404:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091f80. 00010000:00000001:0.0:1788572472.946545:0:261404:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790935552 : -109714918616064 : ffff9c37013fd400) 00010000:00000001:0.0:1788572472.946548:0:261404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.946550:0:261404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.946554:0:261404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000543efac1/0x144158609316868a lrc: 3/1,0 mode: --/PR res: [0x280000401:0x527c:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261404 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.946565:0:261404:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000543efac1/0x144158609316868a lrc: 3/1,0 mode: --/PR res: [0x280000401:0x527c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261404 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.946575:0:261404:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.946578:0:261404:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.946581:0:261404:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.946589:0:261404:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4e00 x1875440809949440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572472.946991:0:261404:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572472.946994:0:261404:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.946996:0:261404:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.946998:0:261404:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.947000:0:261404:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.947003:0:261404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572472.947005:0:261404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572472.947006:0:261404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572472.947008:0:261404:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000800:00000200:0.0:1788572472.947465:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.947470:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572472.947475:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.947480:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.947485:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3088 00000400:00000010:0.0:1788572472.947488:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3088. 00000100:00000001:0.0:1788572472.947492:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.947496:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4e00 x1875440809949440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.947506:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.947510:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440809949440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.947522:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.947524:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572472.954551:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.954567:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572472.954569:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.954572:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.954581:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.954589:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572472.954599:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.954604:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.954610:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.954614:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463900 00000400:00000200:0.0:1788572472.954620:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.954626:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529f11 [1] + 224 00000800:00000001:0.0:1788572472.954654:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.954662:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572472.954666:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.954671:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572472.954679:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.954688:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4e00 x1875440809949440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572472.954700:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c372d6b4e00 x1875440809949440/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.954721:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572472.954729:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572472.954733:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572472.955529:0:261404:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.955533:0:261404:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.955540:0:261404:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572472.955543:0:261404:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c2d8 -> ffff9c3730f6c2d8 00000020:00000001:0.0:1788572472.955545:0:261404:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c2d8 -> ffff9c3730f6c2d8 00000020:00000001:0.0:1788572472.955547:0:261404:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572472.955552:0:261404:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572472.955557:0:261404:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.955559:0:261404:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.955560:0:261404:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572472.955561:0:261404:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.955564:0:261404:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdd6:11031 on OST[0]: s=0 (1) m=1788572467 a=1788572467 c=1788572467 b=0 00020000:00000001:0.0:1788572472.955567:0:261404:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.955569:0:261404:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572467 a=1788572467 c=1788572467 b=0 00020000:00000001:0.0:1788572472.955572:0:261404:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.955574:0:261404:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.955575:0:261404:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c2d8 -> ffff9c3730f6c2d8 00000080:00200000:0.0:1788572472.955578:0:261404:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdd6:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572472.955580:0:261404:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572472.955583:0:261404:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572472.955585:0:261404:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572472.955588:0:261404:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.955589:0:261404:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572472.955593:0:261404:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572472.955597:0:261404:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.955599:0:261404:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293056240 : -109714416495376 : ffff9c371f2d94f0) 00000020:00000001:0.0:1788572472.955601:0:261404:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572472.955602:0:261404:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c372d676f30 (ffff9c3718da6878/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572472.955605:0:261404:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572472.955607:0:261404:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572472.955608:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.955609:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572472.955611:0:261404:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572472.955612:0:261404:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.955614:0:261404:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.955615:0:261404:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572472.955616:0:261404:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.955617:0:261404:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c371ff50b90 (ffff9c3718da6bd8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572472.955620:0:261404:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572472.955622:0:261404:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572472.955623:0:261404:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c372d676f30 (ffff9c3718da6bd8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572472.955626:0:261404:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c37030a6980. 00000020:00000001:0.0:1788572472.955629:0:261404:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572472.955632:0:261404:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c372d676f00. 00020000:00000001:0.0:1788572472.955658:0:261404:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.955660:0:261404:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572472.955661:0:261404:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572472.955663:0:261404:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.955666:0:261404:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572472.955669:0:261404:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572472.955671:0:261404:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572472.955672:0:261404:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572472.955674:0:261404:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.955675:0:261404:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00020000:00000001:0.0:1788572472.955678:0:261404:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572472.955680:0:261404:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572472.955683:0:261404:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572472.955685:0:261404:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdd6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572472.955689:0:261404:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.955691:0:261404:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.955693:0:261404:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572472.955694:0:261404:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572472.955695:0:261404:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572472.955696:0:261404:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6bd0 00000001:00000001:0.0:1788572472.955699:0:261404:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.955706:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbd08 00000080:00000008:0.0:1788572472.955708:0:261404:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-066:[0x200002b17:0xdd6:0x0](ffff9c3708765b88), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.955714:0:261404:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.955716:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbda0 00000080:00002000:0.0:1788572472.955718:0:261404:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-066 (ffff9c37061cc870, parent ffff9c3719d93af8, inode ffff9c3708765b88) hashed, 00000080:00000001:0.0:1788572472.955721:0:261404:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.955781:0:261404:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.955784:0:261404:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.955786:0:261404:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.955788:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbc0 00000080:00200000:0.0:1788572472.955790:0:261404:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.955797:0:261404:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7c00. 00800000:00000002:0.0:1788572472.955803:0:261404:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.955806:0:261404:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.955811:0:261404:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.955813:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbae0 00800000:00000002:0.0:1788572472.955815:0:261404:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.955819:0:261404:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.955821:0:261404:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.955824:0:261404:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.955826:0:261404:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.955828:0:261404:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.955830:0:261404:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.955832:0:261404:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.955834:0:261404:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.955838:0:261404:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.955842:0:261404:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.955844:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb9b0 00000002:00010000:0.0:1788572472.955845:0:261404:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.955848:0:261404:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.955850:0:261404:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.955852:0:261404:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.955858:0:261404:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.955861:0:261404:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.955865:0:261404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.955868:0:261404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.955872:0:261404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 02000000:00000001:0.0:1788572472.955882:0:261404:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.955897:0:261404:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.955900:0:261404:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00000002:00000001:0.0:1788572472.955907:0:261404:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.955908:0:261404:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.955910:0:261404:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.955912:0:261404:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.955913:0:261404:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.955915:0:261404:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.955918:0:261404:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.955921:0:261404:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.955924:0:261404:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7c00. 00000080:00000001:0.0:1788572472.955927:0:261404:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.955929:0:261404:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.955931:0:261404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.955934:0:261404:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.955936:0:261404:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.955938:0:261404:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.955939:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.955941:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.955943:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.955945:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.955946:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.955949:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.955950:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.955951:0:261404:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.955952:0:261404:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.955954:0:261404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20fbbf8 00010000:00000001:0.0:1788572472.955956:0:261404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.955957:0:261404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.955958:0:261404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.955959:0:261404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.955961:0:261404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.955963:0:261404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.955965:0:261404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00010000:00000001:0.0:1788572472.955970:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.955970:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.955971:0:261404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.955973:0:261404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.955975:0:261404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00000001:0.0:1788572472.955986:0:261404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.955988:0:261404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.955990:0:261404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.955992:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.955993:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.955995:0:261404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.955998:0:261404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.955999:0:261404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.956001:0:261404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.956004:0:261404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.956006:0:261404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.956007:0:261404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.956008:0:261404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.956010:0:261404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.956012:0:261404:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.956014:0:261404:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.956018:0:261404:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.956023:0:261404:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.956025:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbb88 00000080:00002000:0.0:1788572472.956027:0:261404:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 66 00000080:00000001:0.0:1788572472.956029:0:261404:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.956032:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbd0 00000080:00200000:0.0:1788572472.956033:0:261404:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572472.956038:0:261404:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.956043:0:261404:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.956044:0:261404:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.956046:0:261404:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.956049:0:261404:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.956050:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbe8 00000080:00002000:0.0:1788572472.956052:0:261404:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-066(ffff9c37061cc870) flags 0x48000e refc 0 00000080:00000001:0.0:1788572472.956054:0:261404:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.956055:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbc30 00000080:00200000:0.0:1788572472.956057:0:261404:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-066, flags=9 00000080:00000001:0.0:1788572472.956067:0:261404:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572472.956072:0:261404:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572472.956075:0:261404:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572472.956076:0:261404:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbe40 00000080:00002000:0.0:1788572472.956078:0:261404:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-066 (ffff9c37061cc870, parent ffff9c3719d93af8, inode ffff9c3708765b88) hashed, 00000080:00000001:0.0:1788572472.956081:0:261404:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.982415:0:261406:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.982423:0:261406:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572472.982426:0:261406:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.982429:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb20 00000080:00200000:0.0:1788572472.982434:0:261406:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572472.982443:0:261406:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572472.982450:0:261406:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572472.982454:0:261406:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.982460:0:261406:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.982462:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba40 00800000:00000002:0.0:1788572472.982465:0:261406:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.982468:0:261406:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.982471:0:261406:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.982474:0:261406:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.982476:0:261406:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.982478:0:261406:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.982481:0:261406:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.982483:0:261406:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572472.982485:0:261406:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.982489:0:261406:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.982494:0:261406:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.982495:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b910 00000002:00010000:0.0:1788572472.982498:0:261406:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.982501:0:261406:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.982503:0:261406:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.982506:0:261406:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.982514:0:261406:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572472.982517:0:261406:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.982523:0:261406:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.982528:0:261406:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.982533:0:261406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 02000000:00000001:0.0:1788572472.982547:0:261406:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.982554:0:261406:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.982558:0:261406:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000002:00000001:0.0:1788572472.982566:0:261406:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.982569:0:261406:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.982571:0:261406:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.982573:0:261406:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.982575:0:261406:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.982578:0:261406:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572472.982581:0:261406:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.982584:0:261406:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572472.982588:0:261406:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572472.982594:0:261406:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.982596:0:261406:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.982599:0:261406:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572472.982603:0:261406:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572472.982606:0:261406:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.982608:0:261406:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.982610:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.982612:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.982615:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.982618:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.982621:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.982623:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.982625:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.982626:0:261406:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.982628:0:261406:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.982631:0:261406:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bb58 00010000:00000001:0.0:1788572472.982898:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.982900:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.982902:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572472.982905:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.982907:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572472.982909:0:261406:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.982912:0:261406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00000001:0.0:1788572472.982920:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.982921:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.982923:0:261406:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.982925:0:261406:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.982928:0:261406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 233 lru_type: 1 00010000:00000001:0.0:1788572472.982934:0:261406:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.982937:0:261406:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.982940:0:261406:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.982944:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.982945:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.982946:0:261406:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.982949:0:261406:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.982951:0:261406:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.982953:0:261406:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.982956:0:261406:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.982958:0:261406:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.982959:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.982960:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.982963:0:261406:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.982965:0:261406:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.982967:0:261406:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572472.982972:0:261406:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.982976:0:261406:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572472.982978:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00002000:0.0:1788572472.982980:0:261406:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 66 00000080:00000001:0.0:1788572472.982983:0:261406:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.982985:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb30 00000080:00200000:0.0:1788572472.982987:0:261406:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572472.982992:0:261406:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572472.982996:0:261406:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572472.982997:0:261406:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572472.983000:0:261406:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572472.983009:0:261406:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f580. 00000020:00000001:0.0:1788572472.983013:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae8 00000080:00200000:0.0:1788572472.983015:0:261406:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-067, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572472.983018:0:261406:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572472.983020:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b998 00000080:00200000:0.0:1788572472.983022:0:261406:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-067, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572472.983027:0:261406:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572472.983030:0:261406:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.983033:0:261406:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.983035:0:261406:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.983036:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b8b8 00800000:00000002:0.0:1788572472.983038:0:261406:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-067' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.983041:0:261406:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.983044:0:261406:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.983046:0:261406:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.983047:0:261406:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.983048:0:261406:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.983050:0:261406:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.983051:0:261406:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.983053:0:261406:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.983055:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b850 00800000:00000002:0.0:1788572472.983057:0:261406:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-067' -> mds #0 00000002:00000001:0.0:1788572472.983060:0:261406:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.983061:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b788 00000002:00010000:0.0:1788572472.983063:0:261406:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-067,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572472.983336:0:261406:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572472.983339:0:261406:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572472.983345:0:261406:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572472.983351:0:261406:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572472.983356:0:261406:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.983361:0:261406:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.983364:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.983365:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.983367:0:261406:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.983369:0:261406:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.983371:0:261406:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.983375:0:261406:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.983381:0:261406:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572472.983388:0:261406:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.983393:0:261406:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.983394:0:261406:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.983396:0:261406:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.983397:0:261406:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.983399:0:261406:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572472.983406:0:261406:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00010000:00000001:0.0:1788572472.983410:0:261406:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.983412:0:261406:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572472.983417:0:261406:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572472.983419:0:261406:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.983422:0:261406:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.983425:0:261406:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fd800. 00000020:00000001:0.0:1788572472.983429:0:261406:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.983431:0:261406:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fd800 with handle 0x1441586093168691 to hash 00000020:00000001:0.0:1788572472.983433:0:261406:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.983435:0:261406:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.983438:0:261406:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.983441:0:261406:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.983443:0:261406:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.983446:0:261406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.983453:0:261406:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.983467:0:261406:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261406 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572472.983472:0:261406:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572472.983474:0:261406:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572472.983479:0:261406:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5a80. 00000100:00000001:0.0:1788572472.983482:0:261406:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288715904 : -109714420835712 : ffff9c371eeb5a80) 00000100:00000001:0.0:1788572472.983484:0:261406:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.983486:0:261406:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000001:0.0:1788572472.983490:0:261406:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.983494:0:261406:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.983496:0:261406:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572472.983498:0:261406:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572472.983509:0:261406:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7100 x1875440809949568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.983517:0:261406:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572472.983519:0:261406:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.983523:0:261406:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572472.983525:0:261406:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572472.983527:0:261406:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572472.983531:0:261406:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261406:1875440809949568:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572472.983534:0:261406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572472.983538:0:261406:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572472.983540:0:261406:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.983542:0:261406:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572472.983546:0:261406:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572472.983548:0:261406:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572472.983554:0:261406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572472.983558:0:261406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8790. 00000400:00000010:0.0:1788572472.983565:0:261406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572472.983572:0:261406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809949568, portal 10 00000100:00000001:0.0:1788572472.983574:0:261406:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572472.983575:0:261406:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000040:0.0:1788572472.983581:0:261406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b7100 x1875440809949568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572557 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.983588:0:261406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572472.983590:0:261406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572472.983592:0:261406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572472.983595:0:261406:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809949568, offset 0 00000400:00000200:0.0:1788572472.983600:0:261406:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572472.983607:0:261406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.983613:0:261406:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572472.983618:0:261406:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572472.983622:0:261406:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572472.983625:0:261406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572472.983628:0:261406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572472.983632:0:261406:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539798:539798:256:4294967295] 192.168.202.148@tcp LPNI seq info [539763:539763:8:4294967295] 00000400:00000200:0.0:1788572472.983900:0:261406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572472.983910:0:261406:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572472.983916:0:261406:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572472.983923:0:261406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572472.983929:0:261406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572472.983937:0:261406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572472.983941:0:261406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572472.983958:0:261406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.983962:0:261406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14993 usecs (15 jiffies) 00000400:00000200:0.0:1788572472.983965:0:261406:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572472.983968:0:261406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612005 00000100:00000001:0.0:1788572472.983971:0:261406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.983973:0:261406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612005 00000100:00000001:0.0:1788572472.983975:0:261406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572472.983979:0:261406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.983981:0:261406:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.983983:0:261406:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572472.983985:0:261406:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572472.983988:0:261406:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5a80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572472.983990:0:261406:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572472.983993:0:261406:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1788572472.992928:0:261406:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.992933:0:261406:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.992936:0:261406:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572472.992938:0:261406:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572472.992941:0:261406:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdd7:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572472.992945:0:261406:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572472.992948:0:261406:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x2c0000401:21002 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572472.992950:0:261406:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4b000) owner ffff9c37303d4f80 00020000:00000001:0.0:1788572472.992952:0:261406:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572472.992954:0:261406:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4b000) owner ffff9c37303d4f80 00020000:00000010:0.0:1788572472.992956:0:261406:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572472.992958:0:261406:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572472.992961:0:261406:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572472.992962:0:261406:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd7:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572472.992964:0:261406:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.992966:0:261406:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.992970:0:261406:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572472.992973:0:261406:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b000) owner 0000000000000000/ffff9c37303d4f80 00020000:00000002:0.0:1788572472.992975:0:261406:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c372dbfdc60 addref 2/0 by ffff9c37303d4f80. 00020000:00000002:0.0:1788572472.992977:0:261406:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b000) owner 0000000000000000/ffff9c37303d4f80 00020000:00000001:0.0:1788572472.992979:0:261406:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572472.992981:0:261406:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572472.992983:0:261406:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572472.992985:0:261406:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572472.992986:0:261406:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572472.992989:0:261406:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdd7:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572472.992992:0:261406:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da63f0 00000080:00000001:0.0:1788572472.992994:0:261406:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.992996:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.992997:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.992999:0:261406:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572472.993001:0:261406:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572472.993003:0:261406:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.993006:0:261406:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd7:0x0](ffff9c3708767448) for lock 0x1441586093168691 00800000:00000001:0.0:1788572472.993009:0:261406:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.993011:0:261406:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.993012:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.993013:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.993015:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd800 refcount=3 00000020:00000001:0.0:1788572472.993016:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993018:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993020:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993021:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.993023:0:261406:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.993024:0:261406:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572472.993031:0:261406:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f9870 inode ffff9c3708767448 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572472.993034:0:261406:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572472.993038:0:261406:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd7:0x0](ffff9c3708767448) 00000080:00010000:0.0:1788572472.993042:0:261406:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168691 from it ffffb95bc207bb20 00010000:00000001:0.0:1788572472.993044:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.993045:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.993046:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd800 refcount=3 00000020:00000001:0.0:1788572472.993048:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993049:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993051:0:261406:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.993053:0:261406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.993059:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993061:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.993062:0:261406:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.993065:0:261406:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.993067:0:261406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.993086:0:261406:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.993088:0:261406:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.993090:0:261406:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.993095:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993096:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.993098:0:261406:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993101:0:261406:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.993103:0:261406:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993105:0:261406:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993108:0:261406:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993110:0:261406:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.993111:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993112:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572472.993115:0:261406:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572472.993118:0:261406:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000100:00000001:0.0:1788572472.993126:0:261406:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.993132:0:261406:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7100 x1875440809949568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572557 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.993139:0:261406:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572472.993142:0:261406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572472.993145:0:261406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572472.993148:0:261406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572472.993151:0:261406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572472.993153:0:261406:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572472.993164:0:261406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572472.993168:0:261406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572472.993173:0:261406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572472.993175:0:261406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572472.993177:0:261406:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7100. 00000100:00000001:0.0:1788572472.993180:0:261406:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572472.993181:0:261406:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.993185:0:261406:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572472.993187:0:261406:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc207bb20 released 00000080:00000001:0.0:1788572472.993188:0:261406:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572472.993199:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00200008:0.0:1788572472.993202:0:261406:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-067:[0x200002b17:0xdd7:0x0](ffff9c3708767448), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572472.993208:0:261406:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572472.993210:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc80 00000080:00200000:0.0:1788572472.993211:0:261406:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdd7:0x0](ffff9c3708767448),name=f434.sanity-067 00000080:00000010:0.0:1788572472.993217:0:261406:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572472.993221:0:261406:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572472.993227:0:261406:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572472.993230:0:261406:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.993232:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bba0 00800000:00000002:0.0:1788572472.993234:0:261406:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdd7:0x0] 'f434.sanity-067' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.993237:0:261406:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572472.993240:0:261406:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.993243:0:261406:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.993245:0:261406:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.993248:0:261406:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.993250:0:261406:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.993252:0:261406:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572472.993254:0:261406:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.993256:0:261406:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572472.993257:0:261406:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572472.993258:0:261406:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572472.993259:0:261406:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572472.993261:0:261406:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.993262:0:261406:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdd7:0x0] 00800000:00000001:0.0:1788572472.993263:0:261406:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572472.993266:0:261406:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdd7:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572472.993269:0:261406:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572472.993271:0:261406:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba70 00000002:00010000:0.0:1788572472.993273:0:261406:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdd7:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572472.993275:0:261406:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572472.993277:0:261406:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572472.993278:0:261406:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572472.993283:0:261406:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c373035f200 count: 3 00010000:00000001:0.0:1788572472.993285:0:261406:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572472.993287:0:261406:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.993290:0:261406:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572472.993293:0:261406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572472.993300:0:261406:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572472.993304:0:261406:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.993307:0:261406:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572472.993313:0:261406:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572472.993315:0:261406:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572472.993317:0:261406:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.993318:0:261406:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572472.993321:0:261406:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572472.993323:0:261406:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572472.993326:0:261406:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572472.993327:0:261406:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572472.993330:0:261406:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572472.993333:0:261406:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572472.993335:0:261406:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.993337:0:261406:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd7:0x0](ffff9c3708767448) 00000080:00010000:0.0:1788572472.993340:0:261406:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd7:0x0](ffff9c3708767448) for lock 0x1441586093168691 00800000:00000001:0.0:1788572472.993343:0:261406:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572472.993344:0:261406:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572472.993345:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.993346:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.993347:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd800 refcount=3 00000020:00000001:0.0:1788572472.993349:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993351:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993353:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993355:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572472.993356:0:261406:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572472.993357:0:261406:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572472.993359:0:261406:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168691 from it ffffb95bc207bcb8 00010000:00000001:0.0:1788572472.993360:0:261406:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.993361:0:261406:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.993363:0:261406:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd800 refcount=3 00000020:00000001:0.0:1788572472.993364:0:261406:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993366:0:261406:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790936576 : -109714918615040 : ffff9c37013fd800) 00010000:00000001:0.0:1788572472.993368:0:261406:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572472.993370:0:261406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572472.993376:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993377:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.993378:0:261406:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572472.993379:0:261406:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572472.993382:0:261406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd800/0x1441586093168691 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2792 expref: -99 pid: 261406 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572472.993387:0:261406:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572472.993389:0:261406:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572472.993390:0:261406:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.993391:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993392:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.993393:0:261406:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993395:0:261406:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572472.993396:0:261406:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993398:0:261406:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993400:0:261406:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993402:0:261406:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572472.993403:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993404:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572472.993406:0:261406:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572472.993407:0:261406:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.993410:0:261406:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572472.993413:0:261406:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.993415:0:261406:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572472.993417:0:261406:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572472.993419:0:261406:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.993421:0:261406:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.993424:0:261406:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572472.993425:0:261406:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.993428:0:261406:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186940920 : -109714522610696 : ffff9c3718da63f8) 00000020:00001000:0.0:1788572472.993430:0:261406:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da63f0 00000020:00000001:0.0:1788572472.993435:0:261406:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c9e0 -> ffff9c3730f6c9e0 00000020:00000001:0.0:1788572472.993437:0:261406:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572472.993439:0:261406:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572472.993441:0:261406:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdd7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572472.993444:0:261406:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.993445:0:261406:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572472.993447:0:261406:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572472.993449:0:261406:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572472.993451:0:261406:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572472.993454:0:261406:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdd7:0x0]io ffff9c371ff530f0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572472.993456:0:261406:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b000) owner 0000000000000000/ffff9c37303d4f80 00020000:00000001:0.0:1788572472.993459:0:261406:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572472.993461:0:261406:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572472.993463:0:261406:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572472.993465:0:261406:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993467:0:261406:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572472.993469:0:261406:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572472.993470:0:261406:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993472:0:261406:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.993473:0:261406:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.993474:0:261406:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572472.993476:0:261406:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b000) owner 0000000000000000/ffff9c37303d4f80 00000020:00000001:0.0:1788572472.993478:0:261406:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572472.993480:0:261406:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572472.993481:0:261406:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdd7:0x0] 00000020:00000001:0.0:1788572472.993484:0:261406:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572472.993485:0:261406:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572472.993488:0:261406:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572472.993490:0:261406:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572472.993495:0:261406:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572472.993501:0:261406:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676f00. 00020000:00000002:0.0:1788572472.993503:0:261406:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572472.993505:0:261406:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572472.993507:0:261406:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.993508:0:261406:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572472.993510:0:261406:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572472.993511:0:261406:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942936 : -109714522608680 : ffff9c3718da6bd8) 00000020:00001000:0.0:1788572472.993513:0:261406:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6bd0 00000020:00000001:0.0:1788572472.993516:0:261406:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6cd00 -> ffff9c3730f6c9e0 00000020:00000001:0.0:1788572472.993518:0:261406:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572472.993520:0:261406:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572472.993521:0:261406:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572472.993523:0:261406:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993524:0:261406:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993526:0:261406:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055656 : -109714416495960 : ffff9c371f2d92a8) 00000020:00000001:0.0:1788572472.993528:0:261406:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572472.993532:0:261406:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6980. 00000008:00010000:0.0:1788572472.993535:0:261406:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572472.993537:0:261406:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.993538:0:261406:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572472.993539:0:261406:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676f30, osc lock ffff9c37030a6980, flags 0x400001002 00000020:00000001:0.0:1788572472.993541:0:261406:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993542:0:261406:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993544:0:261406:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531728640 : -109714177822976 : ffff9c372d676f00) 00000020:00000001:0.0:1788572472.993545:0:261406:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572472.993546:0:261406:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572472.993547:0:261406:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993548:0:261406:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993550:0:261406:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572472.993552:0:261406:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572472.993553:0:261406:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.993554:0:261406:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572472.993556:0:261406:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572472.993557:0:261406:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055656 : -109714416495960 : ffff9c371f2d92a8) 00000020:00000001:0.0:1788572472.993558:0:261406:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.993560:0:261406:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572472.993562:0:261406:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572472.993566:0:261406:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572472.993568:0:261406:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.993569:0:261406:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572472.993571:0:261406:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572472.993572:0:261406:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572472.993573:0:261406:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572472.993576:0:261406:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572472.993577:0:261406:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572472.993581:0:261406:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993583:0:261406:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572472.993587:0:261406:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572472.993591:0:261406:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572472.993595:0:261406:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572472.993599:0:261406:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572472.993602:0:261406:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.993604:0:261406:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.993605:0:261406:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.993607:0:261406:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572472.993609:0:261406:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572472.993612:0:261406:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572472.993617:0:261406:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7200. 00000100:00000001:0.0:1788572472.993625:0:261406:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993629:0:261406:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.993631:0:261406:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993632:0:261406:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572472.993633:0:261406:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993663:0:261406:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.993666:0:261406:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00010000:00000001:0.0:1788572472.993669:0:261406:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572472.993672:0:261406:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572472.993675:0:261406:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0500. 00010000:00000010:0.0:1788572472.993679:0:261406:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0d00. 00010000:00000001:0.0:1788572472.993683:0:261406:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572472.993686:0:261406:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fc800. 00000020:00000001:0.0:1788572472.993689:0:261406:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572472.993692:0:261406:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fc800 with handle 0x1441586093168698 to hash 00000020:00000001:0.0:1788572472.993694:0:261406:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572472.993696:0:261406:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000010:0.0:1788572472.993700:0:261406:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091780. 00010000:00000001:0.0:1788572472.993703:0:261406:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.993705:0:261406:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.993707:0:261406:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.993710:0:261406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572472.993720:0:261406:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.993729:0:261406:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.993733:0:261406:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572472.993737:0:261406:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.993743:0:261406:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9424621] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572472.993771:0:261406:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572472.993774:0:261406:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993776:0:261406:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572472.993778:0:261406:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993779:0:261406:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572472.993782:0:261406:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572472.993783:0:261406:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572472.993785:0:261406:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572472.993787:0:261406:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000800:00000200:0.0:1788572472.994267:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572472.994270:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572472.994274:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572472.994277:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572472.994281:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3f68 00000400:00000010:0.0:1788572472.994283:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3f68. 00000100:00000001:0.0:1788572472.994286:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572472.994289:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.994299:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572472.994301:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.994305:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.994306:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572472.999755:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572472.999761:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572472.999769:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572472.999771:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572472.999776:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2cb8. 00000400:00000200:0.0:1788572472.999781:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788572472.999783:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788572472.999786:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572472.999791:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572472.999799:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.999806:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572472.999808:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572472.999810:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572472.999813:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572472.999816:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572472.999817:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572472.999821:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572472.999842:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=36 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572472.999846:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=9 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572472.999849:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572472.999851:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.999854:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572472.999858:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572472.999861:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572472.999864:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572472.999866:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572472.999870:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572472.999875:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572472.999882:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572472.999885:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572472.999895:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572472.999898:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572472.999901:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.999903:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.999907:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=4 00000020:00000001:0.0:1788572472.999909:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.999912:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.999915:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.999916:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.999917:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=5 00000020:00000001:0.0:1788572472.999919:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.999920:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.999924:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572472.999928:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572472.999932:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.999943:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572472.999945:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572472.999947:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572472.999950:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572472.999951:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572472.999953:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=6 00000020:00000001:0.0:1788572472.999954:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572472.999956:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000040:0.0:1788572472.999963:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37013fc800, remote cookie: 0xd9894672d99a2799, flags: 0x1001 00010000:00010000:0.0:1788572472.999969:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572472.999978:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572472.999981:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572472.999983:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572472.999987:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572472.999991:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572472.999997:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x520a:0x0].0x0 (ffff9c37198c0500) refcount = 1 00010000:00000040:0.0:1788572472.999999:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572473.000002:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.000007:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572473.000008:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.000010:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572473.000012:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.000014:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.000018:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572473.000019:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000020:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.000021:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000022:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.000023:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572473.000025:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572473.000027:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.000028:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.000029:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=5 00000020:00000001:0.0:1788572473.000030:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572473.000031:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00000008:00000001:0.0:1788572473.000034:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572473.000038:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6d880 -> ffff9c3730f6c9e0 00000008:00010000:0.0:1788572473.000040:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572473.000045:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6d880 -> ffff9c3730f6c9e0 00000020:00000001:0.0:1788572473.000047:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572473.000049:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572473.000051:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572473.000053:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.000054:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.000055:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6d880 -> ffff9c3730f6c9e0 00000008:00000001:0.0:1788572473.000056:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572473.000057:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572473.000059:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572473.000072:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572473.000074:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000084:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.000086:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.000088:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=6 00000020:00000001:0.0:1788572473.000089:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572473.000091:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572473.000094:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.000096:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.000103:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000104:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572473.000107:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.000113:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.000115:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000116:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.000117:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000119:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.000120:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.000121:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fc800 refcount=5 00000020:00000001:0.0:1788572473.000123:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572473.000125:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790932480 : -109714918619136 : ffff9c37013fc800) 00010000:00000001:0.0:1788572473.000127:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.000129:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.000136:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000137:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.000138:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.000140:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.000142:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 0000000042104ecf/0x1441586093168698 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x520a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a2799 expref: -99 pid: 261406 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572473.000150:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.000152:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.000154:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.000158:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000159:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.000161:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000164:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.000166:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000168:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000170:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.000172:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.000173:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000174:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.000175:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.000176:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.000178:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.000183:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572473.000192:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809949696:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.000198:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.000201:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.000205:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7100 x1875440809949696/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572488 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.000212:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572473.000215:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572473.000218:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe95c00. 02000000:00000001:0.0:1788572473.000224:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572473.000226:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572473.000228:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572473.000230:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572473.000233:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572473.000238:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572473.000240:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572473.000242:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7100. 00000100:00000001:0.0:1788572473.000245:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572473.000246:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.000248:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.000255:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.000257:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.000259:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.000261:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.016293:0:261408:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.016300:0:261408:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.016302:0:261408:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.016305:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abb20 00000080:00200000:0.0:1788572473.016310:0:261408:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.016317:0:261408:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.016322:0:261408:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.016326:0:261408:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.016331:0:261408:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.016333:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20aba40 00800000:00000002:0.0:1788572473.016335:0:261408:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.016340:0:261408:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.016343:0:261408:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.016345:0:261408:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.016347:0:261408:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.016349:0:261408:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.016352:0:261408:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.016354:0:261408:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.016356:0:261408:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.016359:0:261408:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.016364:0:261408:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.016366:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab910 00000002:00010000:0.0:1788572473.016368:0:261408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.016371:0:261408:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.016373:0:261408:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.016376:0:261408:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.016382:0:261408:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.016385:0:261408:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.016389:0:261408:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.016393:0:261408:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.016398:0:261408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 02000000:00000001:0.0:1788572473.016409:0:261408:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.016413:0:261408:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.016417:0:261408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00000002:00000001:0.0:1788572473.016424:0:261408:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.016425:0:261408:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.016427:0:261408:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.016429:0:261408:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.016431:0:261408:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.016433:0:261408:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.016436:0:261408:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.016439:0:261408:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.016442:0:261408:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572473.016447:0:261408:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.016449:0:261408:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.016452:0:261408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.016455:0:261408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.016458:0:261408:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.016460:0:261408:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.016462:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.016464:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.016467:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.016469:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.016471:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.016473:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.016475:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.016476:0:261408:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.016478:0:261408:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.016481:0:261408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20abb58 00010000:00000001:0.0:1788572473.016483:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.016484:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.016485:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.016487:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.016489:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.016491:0:261408:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.016494:0:261408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00010000:00000001:0.0:1788572473.016500:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.016501:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.016503:0:261408:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.016504:0:261408:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.016507:0:261408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 235 lru_type: 1 00010000:00000001:0.0:1788572473.016519:0:261408:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.016522:0:261408:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.016524:0:261408:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.016528:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.016528:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.016530:0:261408:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016533:0:261408:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.016535:0:261408:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016537:0:261408:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016540:0:261408:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016542:0:261408:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.016543:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.016544:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.016547:0:261408:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.016548:0:261408:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.016550:0:261408:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.016554:0:261408:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.016557:0:261408:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.016559:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abae8 00000080:00002000:0.0:1788572473.016561:0:261408:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 67 00000080:00000001:0.0:1788572473.016564:0:261408:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.016566:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abb30 00000080:00200000:0.0:1788572473.016567:0:261408:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572473.016572:0:261408:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.016575:0:261408:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.016576:0:261408:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.016579:0:261408:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.016585:0:261408:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fb80. 00000020:00000001:0.0:1788572473.016588:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abae8 00000080:00200000:0.0:1788572473.016590:0:261408:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-068, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.016592:0:261408:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.016594:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab998 00000080:00200000:0.0:1788572473.016596:0:261408:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-068, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.016602:0:261408:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.016605:0:261408:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.016608:0:261408:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.016609:0:261408:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.016611:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab8b8 00800000:00000002:0.0:1788572473.016612:0:261408:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-068' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.016615:0:261408:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.016618:0:261408:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.016619:0:261408:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.016620:0:261408:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.016622:0:261408:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.016623:0:261408:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.016625:0:261408:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.016627:0:261408:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.016628:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab850 00800000:00000002:0.0:1788572473.016630:0:261408:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-068' -> mds #0 00000002:00000001:0.0:1788572473.016632:0:261408:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.016634:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab788 00000002:00010000:0.0:1788572473.016652:0:261408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-068,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.016655:0:261408:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.016657:0:261408:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.016661:0:261408:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572473.016665:0:261408:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.016670:0:261408:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.016673:0:261408:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.016676:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.016677:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.016678:0:261408:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.016681:0:261408:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.016683:0:261408:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.016685:0:261408:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.016691:0:261408:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572473.016698:0:261408:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016703:0:261408:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.016704:0:261408:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016706:0:261408:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.016707:0:261408:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.016709:0:261408:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.016715:0:261408:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00010000:00000001:0.0:1788572473.016719:0:261408:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.016722:0:261408:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.016725:0:261408:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.016727:0:261408:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.016729:0:261408:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.016733:0:261408:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fce00. 00000020:00000001:0.0:1788572473.016736:0:261408:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.016738:0:261408:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fce00 with handle 0x144158609316869f to hash 00000020:00000001:0.0:1788572473.016740:0:261408:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.016741:0:261408:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.016745:0:261408:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.016748:0:261408:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.016749:0:261408:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.016753:0:261408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.016760:0:261408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.016770:0:261408:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261408 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.016775:0:261408:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.016777:0:261408:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.016783:0:261408:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5d80. 00000100:00000001:0.0:1788572473.016785:0:261408:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716672 : -109714420834944 : ffff9c371eeb5d80) 00000100:00000001:0.0:1788572473.016787:0:261408:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.016789:0:261408:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000001:0.0:1788572473.016792:0:261408:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.016796:0:261408:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.016798:0:261408:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.016800:0:261408:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.016810:0:261408:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.016820:0:261408:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.016822:0:261408:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.016825:0:261408:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.016828:0:261408:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.016830:0:261408:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.016833:0:261408:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261408:1875440809949824:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.016837:0:261408:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.016842:0:261408:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.016844:0:261408:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.016846:0:261408:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.016850:0:261408:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572473.016852:0:261408:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.016859:0:261408:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.016862:0:261408:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8bb0. 00000400:00000010:0.0:1788572473.016869:0:261408:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572473.016875:0:261408:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809949824, portal 10 00000100:00000001:0.0:1788572473.016877:0:261408:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.016878:0:261408:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000040:0.0:1788572473.016883:0:261408:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.016904:0:261408:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.016907:0:261408:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.016910:0:261408:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572473.016912:0:261408:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809949824, offset 0 00000400:00000200:0.0:1788572473.016919:0:261408:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.016926:0:261408:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.016934:0:261408:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.016938:0:261408:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.016942:0:261408:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.016946:0:261408:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.016949:0:261408:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.016953:0:261408:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539800:539800:256:4294967295] 192.168.202.148@tcp LPNI seq info [539765:539765:8:4294967295] 00000400:00000200:0.0:1788572473.016958:0:261408:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.016967:0:261408:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.016973:0:261408:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.016978:0:261408:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572473.016984:0:261408:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.016992:0:261408:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.016996:0:261408:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.019140:0:261408:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.019152:0:261408:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14992 usecs (15 jiffies) 00000400:00000200:0.0:1788572473.019161:0:261408:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.019164:0:261408:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612040 00000100:00000001:0.0:1788572473.019166:0:261408:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.019168:0:261408:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612040 00000100:00000001:0.0:1788572473.019170:0:261408:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.019173:0:261408:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.019177:0:261408:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.019181:0:261408:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.019183:0:261408:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.019187:0:261408:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5d80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.019191:0:261408:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.019193:0:261408:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.029347:0:261408:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572473.029355:0:261408:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572473.029357:0:261408:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572473.029361:0:261408:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8bb0. 00000400:00000200:0.0:1788572473.029365:0:261408:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed313b8 00000400:00000010:0.0:1788572473.029368:0:261408:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed313b8. 00000100:00000001:0.0:1788572473.029371:0:261408:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.029376:0:261408:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572473.029391:0:261408:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.029398:0:261408:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572473.029400:0:261408:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.029403:0:261408:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572473.029406:0:261408:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572473.029408:0:261408:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572473.029411:0:261408:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.029415:0:261408:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572473.029425:0:261408:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=37 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572473.029430:0:261408:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=69 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572473.029433:0:261408:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572473.029435:0:261408:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.029438:0:261408:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572473.029441:0:261408:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.029445:0:261408:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572473.029447:0:261408:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859757 00000100:00000001:0.0:1788572473.029450:0:261408:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572473.029452:0:261408:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.029458:0:261408:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.029466:0:261408:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.029469:0:261408:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572473.029471:0:261408:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572473.029476:0:261408:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572558 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572473.029485:0:261408:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261408:1875440809949824:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.029491:0:261408:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.029494:0:261408:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.029497:0:261408:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572473.029499:0:261408:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.029503:0:261408:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572558 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.029511:0:261408:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572473.029513:0:261408:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371eeb5d80. 00000100:00000001:0.0:1788572473.029517:0:261408:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572473.029518:0:261408:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.029521:0:261408:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572473.029525:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029528:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029531:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=4 00000020:00000001:0.0:1788572473.029533:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029536:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000040:0.0:1788572473.029542:0:261408:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37013fce00, remote cookie: 0xd9894672d99a27b5, flags: 0x1001 00010000:00010000:0.0:1788572473.029547:0:261408:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572473.029556:0:261408:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdd8:0x0].0x0, instead of [0x200002b17:0xd94:0x0].0x0 00010000:00000001:0.0:1788572473.029560:0:261408:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000040:0.0:1788572473.029566:0:261408:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c373035f800 count: 2 00010000:00000001:0.0:1788572473.029569:0:261408:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.029572:0:261408:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.029576:0:261408:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.029584:0:261408:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029591:0:261408:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572473.029597:0:261408:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572473.029600:0:261408:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572473.029602:0:261408:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572473.029603:0:261408:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572473.029607:0:261408:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdd8:0x0].0x0 (ffff9c373035f800) refcount = 2 00010000:00000040:0.0:1788572473.029611:0:261408:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572473.029613:0:261408:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372007fa00/0x144158609316807f lrc: 1/0,0 mode: CR/CR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a1594 expref: -99 pid: 261255 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572473.029621:0:261408:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029627:0:261408:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572473.029630:0:261408:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572473.029631:0:261408:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.029682:0:261408:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572473.029685:0:261408:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.029689:0:261408:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029696:0:261408:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572473.029697:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029699:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.029700:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029701:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.029703:0:261408:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.029707:0:261408:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572473.029709:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029710:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029712:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.029715:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029717:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029719:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029720:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029721:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=4 00000020:00000001:0.0:1788572473.029723:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029725:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029727:0:261408:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.029731:0:261408:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.029734:0:261408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029740:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029742:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.029743:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029744:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029745:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=5 00000020:00000001:0.0:1788572473.029747:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029749:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029752:0:261408:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.029754:0:261408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029760:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029761:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572473.029764:0:261408:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029769:0:261408:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.029771:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029772:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.029773:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029774:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572473.029779:0:261408:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572558 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572473.029788:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029789:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029791:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.029792:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029794:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00000002:00010000:0.0:1788572473.029798:0:261408:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572473.029830:0:261408:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372d0916c0. 00010000:00000001:0.0:1788572473.029833:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029834:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.029836:0:261408:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.029838:0:261408:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.029841:0:261408:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572473.029843:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029844:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029845:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.029847:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029849:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00000002:00010000:0.0:1788572473.029852:0:261408:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.029858:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029859:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.029861:0:261408:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572473.029862:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.029863:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.029864:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.029866:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.029868:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000040:0.0:1788572473.029871:0:261408:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c373035f800 count: 3 00010000:00000001:0.0:1788572473.029874:0:261408:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572473.029878:0:261408:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3544 (0 0) 00010000:00000001:0.0:1788572473.029881:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.029882:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.029883:0:261408:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572473.029886:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20ab710 00000002:00002000:0.0:1788572473.029899:0:261408:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f434.sanity-068 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572473.029902:0:261408:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.029907:0:261408:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.029910:0:261408:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572473.029914:0:261408:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572473.029917:0:261408:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.029922:0:261408:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.029925:0:261408:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572473.029927:0:261408:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc20abb20 it_disposition b 00000080:00000001:0.0:1788572473.029931:0:261408:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572473.029937:0:261408:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572473.029940:0:261408:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572473.029943:0:261408:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329112 : 144115373145329112 : 200002b17000dd8) 00000080:00000001:0.0:1788572473.029946:0:261408:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572473.029953:0:261408:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.029958:0:261408:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572473.029960:0:261408:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572473.029962:0:261408:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572473.029967:0:261408:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572473.029969:0:261408:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572473.029976:0:261408:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572473.029980:0:261408:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572473.029987:0:261408:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626600. 00020000:00000001:0.0:1788572473.029989:0:261408:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636608 : -109714647915008 : ffff9c3711626600) 00020000:00000002:0.0:1788572473.029993:0:261408:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626600, objid 0xdd8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572473.029998:0:261408:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572473.030003:0:261408:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21117 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572473.030006:0:261408:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4b2e8) owner ffff9c3704cc4f80 00020000:00000001:0.0:1788572473.030010:0:261408:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572473.030012:0:261408:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4b2e8) owner ffff9c3704cc4f80 00020000:00000010:0.0:1788572473.030017:0:261408:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572473.030020:0:261408:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c37116268a0. 00020000:00000010:0.0:1788572473.030023:0:261408:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626600. 00020000:00000002:0.0:1788572473.030025:0:261408:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd8:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572473.030027:0:261408:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030029:0:261408:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572473.030033:0:261408:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00000080:00000001:0.0:1788572473.030037:0:261408:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329112 : 144115373145329112 : 200002b17000dd8) 00000080:00000001:0.0:1788572473.030040:0:261408:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572473.030045:0:261408:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdd8:0x0](ffff9c3708764b08): rc = 0 00000080:00000001:0.0:1788572473.030048:0:261408:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911945480 : -109714797606136 : ffff9c3708764b08) 00010000:00000001:0.0:1788572473.030050:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.030051:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.030053:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.030055:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030057:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00000080:00000001:0.0:1788572473.030060:0:261408:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572473.030062:0:261408:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.030063:0:261408:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572473.030065:0:261408:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572473.030067:0:261408:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572473.030068:0:261408:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572473.030069:0:261408:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572473.030072:0:261408:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572473.030075:0:261408:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572473.030077:0:261408:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c37116268a0. 00020000:00000001:0.0:1788572473.030079:0:261408:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061637280 : -109714647914336 : ffff9c37116268a0) 00020000:00000002:0.0:1788572473.030081:0:261408:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c37116268a0, objid 0xdd8:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572473.030085:0:261408:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572473.030088:0:261408:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c3730106700: ostid: 0x280000401:21117 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572473.030090:0:261408:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c371fe4b2e8) owner ffff9c3704cc4f80 00020000:00000001:0.0:1788572473.030092:0:261408:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572473.030094:0:261408:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c371fe4b2e8) owner ffff9c3704cc4f80 00020000:00000010:0.0:1788572473.030096:0:261408:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c3730106700. 00020000:00000010:0.0:1788572473.030098:0:261408:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626600. 00020000:00000010:0.0:1788572473.030100:0:261408:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c37116268a0. 00020000:00000002:0.0:1788572473.030101:0:261408:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdd8:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572473.030103:0:261408:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030105:0:261408:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030108:0:261408:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572473.030111:0:261408:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b2e8) owner 0000000000000000/ffff9c3704cc4f80 00020000:00000002:0.0:1788572473.030113:0:261408:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c372dbfd0c0 addref 2/0 by ffff9c3704cc4f80. 00020000:00000002:0.0:1788572473.030116:0:261408:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b2e8) owner 0000000000000000/ffff9c3704cc4f80 00020000:00000001:0.0:1788572473.030118:0:261408:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572473.030120:0:261408:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572473.030122:0:261408:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572473.030123:0:261408:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572473.030125:0:261408:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572473.030128:0:261408:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdd8:0x0]: layout version change: 0 -> 0 00000020:00001000:0.0:1788572473.030130:0:261408:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6870 00000080:00000001:0.0:1788572473.030132:0:261408:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030134:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030135:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572473.030137:0:261408:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572473.030140:0:261408:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572473.030141:0:261408:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.030144:0:261408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd8:0x0](ffff9c3708764b08) for lock 0x144158609316869f 00800000:00000001:0.0:1788572473.030148:0:261408:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.030149:0:261408:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.030151:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.030152:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.030154:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.030155:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030157:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030159:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030160:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.030162:0:261408:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.030163:0:261408:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572473.030170:0:261408:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37083f9288 inode ffff9c3708764b08 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572473.030174:0:261408:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572473.030177:0:261408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd8:0x0](ffff9c3708764b08) 00000080:00010000:0.0:1788572473.030181:0:261408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316869f from it ffffb95bc20abb20 00010000:00000001:0.0:1788572473.030183:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.030185:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.030186:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.030188:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030190:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030192:0:261408:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.030195:0:261408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.030202:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030203:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.030204:0:261408:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.030206:0:261408:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.030209:0:261408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572473.030216:0:261408:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.030218:0:261408:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.030221:0:261408:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.030225:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030226:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.030228:0:261408:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030231:0:261408:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.030233:0:261408:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030235:0:261408:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030238:0:261408:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030239:0:261408:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.030241:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030242:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572473.030243:0:261408:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572473.030247:0:261408:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000100:00000001:0.0:1788572473.030254:0:261408:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.030259:0:261408:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7100 x1875440809949824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/640 e 0 to 0 dl 1788572558 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.030266:0:261408:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572473.030268:0:261408:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572473.030272:0:261408:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572473.030275:0:261408:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572473.030278:0:261408:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572473.030280:0:261408:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=42 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572473.030282:0:261408:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572473.030285:0:261408:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572473.030288:0:261408:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572473.030290:0:261408:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572473.030293:0:261408:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7100. 00000100:00000001:0.0:1788572473.030296:0:261408:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572473.030297:0:261408:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.030301:0:261408:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572473.030302:0:261408:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc20abb20 released 00000080:00000001:0.0:1788572473.030304:0:261408:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572473.030315:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abd08 00000080:00200008:0.0:1788572473.030318:0:261408:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-068:[0x200002b17:0xdd8:0x0](ffff9c3708764b08), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.030324:0:261408:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.030326:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abc80 00000080:00200000:0.0:1788572473.030328:0:261408:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdd8:0x0](ffff9c3708764b08),name=f434.sanity-068 00000080:00000010:0.0:1788572473.030334:0:261408:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.030339:0:261408:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.030345:0:261408:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.030348:0:261408:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.030349:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20abba0 00800000:00000002:0.0:1788572473.030352:0:261408:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdd8:0x0] 'f434.sanity-068' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.030355:0:261408:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.030359:0:261408:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.030362:0:261408:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.030364:0:261408:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.030367:0:261408:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.030369:0:261408:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.030371:0:261408:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.030373:0:261408:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.030375:0:261408:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.030376:0:261408:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.030377:0:261408:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.030379:0:261408:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.030380:0:261408:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.030382:0:261408:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdd8:0x0] 00800000:00000001:0.0:1788572473.030384:0:261408:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.030386:0:261408:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdd8:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.030391:0:261408:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.030392:0:261408:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20aba70 00000002:00010000:0.0:1788572473.030394:0:261408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdd8:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.030398:0:261408:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.030399:0:261408:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.030401:0:261408:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.030404:0:261408:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c373035f800 count: 3 00010000:00000001:0.0:1788572473.030406:0:261408:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.030409:0:261408:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.030411:0:261408:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.030414:0:261408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572473.030422:0:261408:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.030426:0:261408:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.030430:0:261408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572473.030437:0:261408:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.030439:0:261408:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.030441:0:261408:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.030443:0:261408:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.030445:0:261408:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.030447:0:261408:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572473.030449:0:261408:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.030451:0:261408:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572473.030453:0:261408:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572473.030456:0:261408:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.030458:0:261408:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.030460:0:261408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdd8:0x0](ffff9c3708764b08) 00000080:00010000:0.0:1788572473.030464:0:261408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdd8:0x0](ffff9c3708764b08) for lock 0x144158609316869f 00800000:00000001:0.0:1788572473.030466:0:261408:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.030468:0:261408:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.030469:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.030470:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.030472:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.030474:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030477:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030479:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030480:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.030481:0:261408:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.030483:0:261408:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.030485:0:261408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316869f from it ffffb95bc20abcb8 00010000:00000001:0.0:1788572473.030487:0:261408:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.030489:0:261408:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.030491:0:261408:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fce00 refcount=3 00000020:00000001:0.0:1788572473.030492:0:261408:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030494:0:261408:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934016 : -109714918617600 : ffff9c37013fce00) 00010000:00000001:0.0:1788572473.030496:0:261408:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.030499:0:261408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572473.030506:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030507:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.030508:0:261408:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.030510:0:261408:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.030512:0:261408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fce00/0x144158609316869f lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdd8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a27b5 expref: -99 pid: 261408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572473.030519:0:261408:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.030521:0:261408:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.030522:0:261408:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.030524:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030526:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.030527:0:261408:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030529:0:261408:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.030530:0:261408:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030532:0:261408:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030534:0:261408:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030536:0:261408:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.030537:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030539:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.030540:0:261408:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.030542:0:261408:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.030545:0:261408:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572473.030548:0:261408:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572473.030550:0:261408:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.030552:0:261408:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572473.030554:0:261408:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.030555:0:261408:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572473.030559:0:261408:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572473.030561:0:261408:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.030562:0:261408:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186942072 : -109714522609544 : ffff9c3718da6878) 00000020:00001000:0.0:1788572473.030565:0:261408:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6870 00000020:00000001:0.0:1788572473.030569:0:261408:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6c850 -> ffff9c3730f6c850 00000020:00000001:0.0:1788572473.030572:0:261408:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572473.030574:0:261408:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572473.030576:0:261408:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdd8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572473.030580:0:261408:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.030581:0:261408:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572473.030583:0:261408:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.030585:0:261408:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.030587:0:261408:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.030590:0:261408:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdd8:0x0]io ffff9c371ff537e0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572473.030594:0:261408:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fe4b2e8) owner 0000000000000000/ffff9c3704cc4f80 00020000:00000001:0.0:1788572473.030597:0:261408:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572473.030598:0:261408:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572473.030600:0:261408:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572473.030602:0:261408:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030604:0:261408:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572473.030607:0:261408:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572473.030608:0:261408:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030610:0:261408:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.030612:0:261408:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572473.030613:0:261408:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572473.030614:0:261408:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fe4b2e8) owner 0000000000000000/ffff9c3704cc4f80 00000020:00000001:0.0:1788572473.030617:0:261408:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572473.030619:0:261408:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572473.030621:0:261408:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdd8:0x0] 00000020:00000001:0.0:1788572473.030623:0:261408:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572473.030625:0:261408:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572473.030627:0:261408:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572473.030629:0:261408:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572473.030691:0:261408:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572473.030696:0:261408:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676380. 00020000:00000002:0.0:1788572473.030699:0:261408:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572473.030701:0:261408:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572473.030703:0:261408:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.030705:0:261408:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572473.030706:0:261408:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.030708:0:261408:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887899432 : -109714821652184 : ffff9c3707076128) 00000020:00001000:0.0:1788572473.030710:0:261408:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3707076120 00000020:00000001:0.0:1788572473.030713:0:261408:0:(cl_object.c:143:cl_object_top()) ffff9c3730f6cdc8 -> ffff9c3730f6c850 00000020:00000001:0.0:1788572473.030715:0:261408:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572473.030719:0:261408:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.030719:0:261408:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572473.030721:0:261408:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030722:0:261408:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030724:0:261408:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055072 : -109714416496544 : ffff9c371f2d9060) 00000020:00000001:0.0:1788572473.030726:0:261408:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572473.030729:0:261408:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6390. 00000008:00010000:0.0:1788572473.030732:0:261408:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572473.030734:0:261408:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572473.030735:0:261408:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572473.030737:0:261408:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d6763b0, osc lock ffff9c37030a6390, flags 0x400001002 00000020:00000001:0.0:1788572473.030739:0:261408:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030740:0:261408:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030743:0:261408:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725696 : -109714177825920 : ffff9c372d676380) 00000020:00000001:0.0:1788572473.030745:0:261408:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572473.030745:0:261408:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572473.030748:0:261408:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030749:0:261408:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030751:0:261408:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572473.030753:0:261408:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572473.030755:0:261408:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572473.030756:0:261408:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572473.030757:0:261408:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.030758:0:261408:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293055072 : -109714416496544 : ffff9c371f2d9060) 00000020:00000001:0.0:1788572473.030760:0:261408:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572473.030762:0:261408:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572473.030764:0:261408:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572473.030767:0:261408:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572473.030769:0:261408:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572473.030770:0:261408:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572473.030771:0:261408:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572473.030772:0:261408:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572473.030773:0:261408:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572473.030775:0:261408:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572473.030777:0:261408:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572473.030780:0:261408:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030783:0:261408:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572473.030787:0:261408:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572473.030790:0:261408:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572473.030793:0:261408:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.030798:0:261408:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.030801:0:261408:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.030802:0:261408:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.030803:0:261408:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.030805:0:261408:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.030807:0:261408:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.030810:0:261408:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.030814:0:261408:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7200. 00000100:00000001:0.0:1788572473.030821:0:261408:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030825:0:261408:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.030827:0:261408:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030828:0:261408:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.030830:0:261408:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030832:0:261408:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.030834:0:261408:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00010000:00000001:0.0:1788572473.030837:0:261408:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.030839:0:261408:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572473.030842:0:261408:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c37198c0300. 00010000:00000010:0.0:1788572473.030845:0:261408:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c37198c0f00. 00010000:00000001:0.0:1788572473.030849:0:261408:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.030852:0:261408:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fd000. 00000020:00000001:0.0:1788572473.030856:0:261408:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.030859:0:261408:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fd000 with handle 0x14415860931686a6 to hash 00000020:00000001:0.0:1788572473.030861:0:261408:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.030862:0:261408:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000010:0.0:1788572473.030865:0:261408:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091fc0. 00010000:00000001:0.0:1788572473.030868:0:261408:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.030870:0:261408:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.030872:0:261408:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.030875:0:261408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.030885:0:261408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.030907:0:261408:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.030912:0:261408:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.030916:0:261408:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.030924:0:261408:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9424621] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572473.030948:0:261408:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572473.030951:0:261408:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030953:0:261408:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.030955:0:261408:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030956:0:261408:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.030959:0:261408:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572473.030962:0:261408:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572473.030964:0:261408:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572473.030965:0:261408:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572473.030976:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572473.030979:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572473.030981:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.030988:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.030991:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.030993:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.030994:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.030996:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.031001:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.031008:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.031010:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.031014:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.031017:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.031019:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.031022:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809949952:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.031027:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.031035:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.031037:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.031039:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.031045:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572473.031047:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.031055:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.031059:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8bb0. 00000400:00000010:0.0:1788572473.031066:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572473.031072:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809949952, portal 4 00000100:00000001:0.0:1788572473.031074:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.031076:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000040:0.0:1788572473.031082:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.031089:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.031093:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.031095:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572473.031098:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809949952, offset 0 00000400:00000200:0.0:1788572473.031103:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.031109:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.031116:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.031121:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.031125:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.031129:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.031132:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.031136:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539801:539801:256:4294967295] 192.168.202.148@tcp LPNI seq info [539766:539766:8:4294967295] 00000400:00000200:0.0:1788572473.031140:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.031149:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.031155:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.031161:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572473.031168:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.031177:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.031181:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.031191:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.031195:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 167551 usecs (168 jiffies) 00000400:00000200:0.0:1788572473.031199:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.031202:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612203 00000100:00000001:0.0:1788572473.031205:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.031207:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612203 00000100:00000001:0.0:1788572473.031209:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.031211:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.031214:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.031217:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.031219:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.036568:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.036573:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572473.036579:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572473.036582:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572473.036586:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8bb0. 00000400:00000200:0.0:1788572473.036590:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed313b8 00000400:00000010:0.0:1788572473.036593:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed313b8. 00000100:00000001:0.0:1788572473.036596:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.036601:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572473.036610:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.036617:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572473.036619:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.036622:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572473.036625:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572473.036627:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572473.036629:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.036632:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572473.036687:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=37 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572473.036692:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=88 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572473.036695:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572473.036697:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.036699:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572473.036702:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.036706:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572473.036708:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727877 00000100:00000001:0.0:1788572473.036710:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572473.036713:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.036719:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.036726:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.036728:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572473.036730:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572473.036733:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572473.036736:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036739:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036741:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=4 00000020:00000001:0.0:1788572473.036744:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036747:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036749:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036750:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036751:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=5 00000020:00000001:0.0:1788572473.036753:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036755:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036758:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.036761:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.036766:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036775:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036777:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.036779:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572473.036780:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036781:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036783:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=6 00000020:00000001:0.0:1788572473.036785:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036787:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000040:0.0:1788572473.036793:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c37013fd000, remote cookie: 0xd9894672d99a27bc, flags: 0x1001 00010000:00010000:0.0:1788572473.036797:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036805:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572473.036807:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.036809:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572473.036813:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572473.036817:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572473.036831:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x527d:0x0].0x0 (ffff9c37198c0300) refcount = 1 00010000:00000040:0.0:1788572473.036834:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572473.036836:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036845:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572473.036847:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.036849:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572473.036852:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.036855:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036861:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572473.036863:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036864:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.036865:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036866:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.036868:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572473.036870:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572473.036873:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036874:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036876:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=5 00000020:00000001:0.0:1788572473.036878:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036880:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00000008:00000001:0.0:1788572473.036884:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572473.036896:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6ce00 -> ffff9c3730f6c850 00000008:00010000:0.0:1788572473.036900:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572473.036908:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6ce00 -> ffff9c3730f6c850 00000020:00000001:0.0:1788572473.036910:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572473.036913:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572473.036916:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572473.036918:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.036920:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.036922:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3730d6ce00 -> ffff9c3730f6c850 00000008:00000001:0.0:1788572473.036923:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572473.036926:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572473.036928:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572473.036945:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572473.036946:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.036949:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036950:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036951:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=6 00000020:00000001:0.0:1788572473.036953:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036955:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036957:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.036960:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036966:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036968:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572473.036971:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036976:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.036978:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036978:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.036980:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.036981:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.036982:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.036984:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37013fd000 refcount=5 00000020:00000001:0.0:1788572473.036985:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036987:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358790934528 : -109714918617088 : ffff9c37013fd000) 00010000:00000001:0.0:1788572473.036989:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.036991:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.036997:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.036998:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.037000:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.037002:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.037004:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c54e3f30/0x14415860931686a6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x527d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a27bc expref: -99 pid: 261408 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572473.037011:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.037014:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.037017:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.037021:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.037022:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.037023:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.037026:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.037028:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.037030:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.037032:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.037034:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.037035:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.037036:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.037037:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.037038:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.037039:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.037044:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572473.037052:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809949952:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.037057:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.037060:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.037064:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7100 x1875440809949952/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.037071:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572473.037073:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572473.037077:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572473.037081:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572473.037083:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572473.037085:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572473.037088:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572473.037090:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37101b7200. 02000000:00000001:0.0:1788572473.037096:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572473.037098:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572473.037100:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7100. 00000100:00000001:0.0:1788572473.037103:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572473.037105:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.037107:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.037114:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.037116:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.037118:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.037120:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572473.119678:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.119685:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572473.119691:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.119697:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.119703:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572473.119719:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572473.119723:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.119728:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6a00 x1875440809950336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.119738:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.119741:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6a00 x1875440809950336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.119748:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.119749:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.125112:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.125131:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.125137:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.125140:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.125149:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.125158:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.125168:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.125173:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.125179:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.125183:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463c80 00000400:00000200:0.0:1788572473.125189:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.125195:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529f79 [1] + 224 00000800:00000001:0.0:1788572473.125202:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.125209:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.125213:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.125218:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.125227:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.125236:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6a00 x1875440809950336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.125247:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6a00 x1875440809950336/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.125269:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.125278:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.125280:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.128783:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.128789:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572473.128793:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.128798:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.128802:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c17f8 00000400:00000010:0.0:1788572473.128804:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c17f8. 00000100:00000001:0.0:1788572473.128810:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.128814:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b800 x1875440809950464/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.128823:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.128832:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440809950464/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.128839:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.128841:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.131229:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.131246:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.131251:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.131253:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.131262:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.131271:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.131280:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.131286:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.131291:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.131295:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463d00 00000400:00000200:0.0:1788572473.131300:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.131306:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529f89 [1] + 224 00000800:00000001:0.0:1788572473.131313:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.131321:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.131324:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.131328:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.131336:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.131344:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b800 x1875440809950464/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.131355:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6b800 x1875440809950464/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.131371:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.131378:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.131381:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.165900:0:261414:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.165908:0:261414:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.165910:0:261414:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.165914:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213bb20 00000080:00200000:0.0:1788572473.165919:0:261414:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.165927:0:261414:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.165934:0:261414:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.165939:0:261414:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.165943:0:261414:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.165946:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213ba40 00800000:00000002:0.0:1788572473.165948:0:261414:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.165952:0:261414:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.165954:0:261414:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.165957:0:261414:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.165959:0:261414:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.165961:0:261414:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.165964:0:261414:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.165966:0:261414:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.165968:0:261414:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.165972:0:261414:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.165977:0:261414:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.165979:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213b910 00000002:00010000:0.0:1788572473.165981:0:261414:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.165984:0:261414:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.165986:0:261414:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.165989:0:261414:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.165997:0:261414:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.166000:0:261414:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.166006:0:261414:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.166011:0:261414:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.166016:0:261414:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 240 lru_type: 1 02000000:00000001:0.0:1788572473.166028:0:261414:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.166033:0:261414:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.166036:0:261414:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 240 lru_type: 1 00000002:00000001:0.0:1788572473.166042:0:261414:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.166045:0:261414:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.166047:0:261414:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.166049:0:261414:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.166051:0:261414:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.166053:0:261414:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.166056:0:261414:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.166059:0:261414:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.166062:0:261414:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572473.166068:0:261414:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.166070:0:261414:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.166073:0:261414:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.166076:0:261414:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.166079:0:261414:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.166800:0:261414:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.166804:0:261414:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.166806:0:261414:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.166810:0:261414:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.166813:0:261414:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.166817:0:261414:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.166820:0:261414:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.166822:0:261414:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.166823:0:261414:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.166824:0:261414:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.166827:0:261414:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc213bb58 00010000:00000001:0.0:1788572473.166830:0:261414:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.166831:0:261414:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.166832:0:261414:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.166834:0:261414:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.166836:0:261414:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.166838:0:261414:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.166842:0:261414:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 240 lru_type: 1 00010000:00000001:0.0:1788572473.166850:0:261414:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.166852:0:261414:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.166853:0:261414:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.166856:0:261414:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.166859:0:261414:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 241 lru_type: 1 00010000:00000001:0.0:1788572473.166867:0:261414:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.166870:0:261414:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.166872:0:261414:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.166876:0:261414:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.166877:0:261414:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.166879:0:261414:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.166882:0:261414:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.166884:0:261414:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.166886:0:261414:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.166900:0:261414:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.166902:0:261414:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.166903:0:261414:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.166904:0:261414:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.166908:0:261414:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.166909:0:261414:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.166912:0:261414:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.166917:0:261414:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.166922:0:261414:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.166925:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213bae8 00000080:00002000:0.0:1788572473.166927:0:261414:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 70 00000080:00000001:0.0:1788572473.166929:0:261414:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.166931:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213bb30 00000080:00200000:0.0:1788572473.166932:0:261414:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572473.166938:0:261414:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.166943:0:261414:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.166944:0:261414:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.166946:0:261414:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.166956:0:261414:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f300. 00000020:00000001:0.0:1788572473.166959:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213bae8 00000080:00200000:0.0:1788572473.166962:0:261414:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-071, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.166964:0:261414:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.166967:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213b998 00000080:00200000:0.0:1788572473.166968:0:261414:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-071, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.166975:0:261414:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.166978:0:261414:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.166982:0:261414:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.166984:0:261414:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.166986:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213b8b8 00800000:00000002:0.0:1788572473.166988:0:261414:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-071' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.166991:0:261414:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.166994:0:261414:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.166996:0:261414:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.166998:0:261414:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.167000:0:261414:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.167001:0:261414:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.167003:0:261414:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.167006:0:261414:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.167008:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213b850 00800000:00000002:0.0:1788572473.167010:0:261414:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-071' -> mds #0 00000002:00000001:0.0:1788572473.167013:0:261414:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.167015:0:261414:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc213b788 00000002:00010000:0.0:1788572473.167016:0:261414:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-071,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.167020:0:261414:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.167022:0:261414:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.167027:0:261414:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000020:00000040:0.0:1788572473.167031:0:261414:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.167036:0:261414:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.167041:0:261414:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.167044:0:261414:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.167045:0:261414:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.167046:0:261414:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.167049:0:261414:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.167051:0:261414:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.167054:0:261414:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.167061:0:261414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572473.167069:0:261414:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.167075:0:261414:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.167077:0:261414:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.167078:0:261414:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.167079:0:261414:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.167081:0:261414:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.167200:0:261414:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00010000:00000001:0.0:1788572473.167204:0:261414:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.167206:0:261414:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.167212:0:261414:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.167215:0:261414:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.167218:0:261414:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.167222:0:261414:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fd200. 00000020:00000001:0.0:1788572473.167226:0:261414:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.167229:0:261414:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fd200 with handle 0x14415860931686c9 to hash 00000020:00000001:0.0:1788572473.167231:0:261414:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.167232:0:261414:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790935040 : -109714918616576 : ffff9c37013fd200) 00010000:00000001:0.0:1788572473.167236:0:261414:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790935040 : -109714918616576 : ffff9c37013fd200) 00010000:00000001:0.0:1788572473.167239:0:261414:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.167241:0:261414:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.167245:0:261414:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd200/0x14415860931686c9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261414 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.167253:0:261414:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd200/0x14415860931686c9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261414 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.167264:0:261414:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fd200/0x14415860931686c9 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261414 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.167269:0:261414:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.167271:0:261414:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.167279:0:261414:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5d80. 00000100:00000001:0.0:1788572473.167282:0:261414:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716672 : -109714420834944 : ffff9c371eeb5d80) 00000100:00000001:0.0:1788572473.167285:0:261414:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.167287:0:261414:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000001:0.0:1788572473.167291:0:261414:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.167296:0:261414:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.167298:0:261414:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.167299:0:261414:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.167310:0:261414:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7100 x1875440809950592/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.167319:0:261414:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.167321:0:261414:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.167325:0:261414:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.167328:0:261414:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.167331:0:261414:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.167335:0:261414:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7100 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261414:1875440809950592:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.167338:0:261414:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.167345:0:261414:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.167347:0:261414:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.167349:0:261414:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.167354:0:261414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572473.167356:0:261414:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.167364:0:261414:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.167369:0:261414:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8bb0. 00000400:00000010:0.0:1788572473.167375:0:261414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572473.167382:0:261414:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809950592, portal 10 00000100:00000001:0.0:1788572473.167384:0:261414:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.167385:0:261414:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531991296 : -109714177560320 : ffff9c372d6b7100) 00000100:00000040:0.0:1788572473.167390:0:261414:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b7100 x1875440809950592/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.167398:0:261414:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.167401:0:261414:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.167403:0:261414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572473.167406:0:261414:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809950592, offset 0 00000400:00000200:0.0:1788572473.167413:0:261414:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.167422:0:261414:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.167430:0:261414:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.167436:0:261414:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.167441:0:261414:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.167445:0:261414:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.167448:0:261414:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.167452:0:261414:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539806:539806:256:4294967295] 192.168.202.148@tcp LPNI seq info [539771:539771:8:4294967295] 00000400:00000200:0.0:1788572473.167457:0:261414:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.167466:0:261414:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.167472:0:261414:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.167478:0:261414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572473.167485:0:261414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.167493:0:261414:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.167498:0:261414:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.167568:0:261414:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.167573:0:261414:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 14991 usecs (15 jiffies) 00000400:00000200:0.0:1788572473.167577:0:261414:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.167580:0:261414:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612188 00000100:00000001:0.0:1788572473.167584:0:261414:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.167587:0:261414:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612188 00000100:00000001:0.0:1788572473.167589:0:261414:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.167591:0:261414:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.167594:0:261414:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.167596:0:261414:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.167598:0:261414:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.167600:0:261414:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5d80 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.167603:0:261414:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.167606:0:261414:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.180398:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.180414:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.180418:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.180421:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.180430:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.180437:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.180447:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.180451:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.180456:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.180460:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463d80 00000400:00000200:0.0:1788572473.180466:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.180473:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529f91 [1] + 224 00000800:00000001:0.0:1788572473.180481:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.180535:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.180539:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.180544:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.180551:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.180562:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7100 x1875440809950592/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.180573:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b7100 x1875440809950592/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.180594:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.180603:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.180606:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.193302:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.193322:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.193327:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.193330:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.193341:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.193349:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.193359:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.193364:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.193370:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.193374:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463e00 00000400:00000200:0.0:1788572473.193379:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.193386:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529fa1 [1] + 224 00000800:00000001:0.0:1788572473.193393:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.193400:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.193403:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.193408:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.193416:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.193425:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69500 x1875440809950720/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.193436:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69500 x1875440809950720/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.193459:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.193466:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.193469:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.231502:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.231513:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572473.231521:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.231530:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.231539:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1550 00000400:00000010:0.0:1788572473.231542:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1550. 00000100:00000001:0.0:1788572473.231547:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.231556:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68e00 x1875440809950848/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.231567:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.231571:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68e00 x1875440809950848/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.231578:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.231580:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.260380:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.260405:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.260410:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.260413:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.260421:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.260429:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.260440:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.260445:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.260451:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.260455:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463e80 00000400:00000200:0.0:1788572473.260460:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.260466:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529fa9 [1] + 224 00000800:00000001:0.0:1788572473.260473:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.260485:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.260488:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.260493:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.260500:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.260509:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68e00 x1875440809950848/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.260521:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed68e00 x1875440809950848/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.260632:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.260657:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.260661:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.265340:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.265345:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572473.265350:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.265355:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.265360:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1330 00000400:00000010:0.0:1788572473.265363:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1330. 00000100:00000001:0.0:1788572473.265366:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.265370:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68e00 x1875440809950976/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.265378:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.265381:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440809950976/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.265387:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.265388:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.270278:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.270297:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.270303:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.270305:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.270315:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.270322:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.270331:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.270336:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.270341:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.270345:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417463f00 00000400:00000200:0.0:1788572473.270350:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.270357:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529fb1 [1] + 224 00000800:00000001:0.0:1788572473.270364:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.270383:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.270387:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.270391:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.270399:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.270408:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68e00 x1875440809950976/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.270418:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed68e00 x1875440809950976/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.270440:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.270449:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.270452:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.340904:0:261418:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.340915:0:261418:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.340918:0:261418:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.340922:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123b20 00000080:00200000:0.0:1788572473.340927:0:261418:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.340935:0:261418:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6800. 00800000:00000002:0.0:1788572473.340942:0:261418:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.340947:0:261418:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.340952:0:261418:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.340954:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123a40 00800000:00000002:0.0:1788572473.340956:0:261418:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.340961:0:261418:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.340963:0:261418:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.340966:0:261418:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.340968:0:261418:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.340970:0:261418:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.340972:0:261418:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.340974:0:261418:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.340976:0:261418:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.340979:0:261418:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.340984:0:261418:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.340985:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123910 00000002:00010000:0.0:1788572473.340987:0:261418:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.340991:0:261418:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.340993:0:261418:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.340996:0:261418:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.341003:0:261418:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.341006:0:261418:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.341011:0:261418:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.341014:0:261418:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.341019:0:261418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 244 lru_type: 1 02000000:00000001:0.0:1788572473.341030:0:261418:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.341035:0:261418:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.341038:0:261418:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 244 lru_type: 1 00000002:00000001:0.0:1788572473.341044:0:261418:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.341046:0:261418:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.341048:0:261418:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.341050:0:261418:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.341052:0:261418:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.341054:0:261418:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.341056:0:261418:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.341059:0:261418:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.341063:0:261418:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6800. 00000080:00000001:0.0:1788572473.341070:0:261418:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.341071:0:261418:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.341074:0:261418:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.341077:0:261418:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.341080:0:261418:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.341082:0:261418:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.341084:0:261418:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.341086:0:261418:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.341089:0:261418:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.341091:0:261418:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.341094:0:261418:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.341097:0:261418:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.341099:0:261418:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.341100:0:261418:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.341102:0:261418:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.341104:0:261418:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc2123b58 00010000:00000001:0.0:1788572473.341106:0:261418:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.341108:0:261418:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.341109:0:261418:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.341111:0:261418:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.341113:0:261418:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.341115:0:261418:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.341118:0:261418:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 244 lru_type: 1 00010000:00000001:0.0:1788572473.341125:0:261418:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.341126:0:261418:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.341128:0:261418:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.341130:0:261418:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.341133:0:261418:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 245 lru_type: 1 00010000:00000001:0.0:1788572473.341148:0:261418:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.341151:0:261418:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.341154:0:261418:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.341158:0:261418:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.341160:0:261418:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.341161:0:261418:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341164:0:261418:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.341166:0:261418:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341168:0:261418:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341170:0:261418:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341172:0:261418:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.341173:0:261418:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.341174:0:261418:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.341177:0:261418:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.341179:0:261418:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.341181:0:261418:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.341185:0:261418:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.341188:0:261418:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.341190:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123ae8 00000080:00002000:0.0:1788572473.341192:0:261418:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 72 00000080:00000001:0.0:1788572473.341195:0:261418:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.341197:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123b30 00000080:00200000:0.0:1788572473.341198:0:261418:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572473.341203:0:261418:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.341207:0:261418:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.341209:0:261418:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.341211:0:261418:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.341221:0:261418:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f220. 00000020:00000001:0.0:1788572473.341225:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123ae8 00000080:00200000:0.0:1788572473.341227:0:261418:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-073, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.341230:0:261418:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.341232:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123998 00000080:00200000:0.0:1788572473.341233:0:261418:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-073, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.341239:0:261418:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6800. 00800000:00000002:0.0:1788572473.341242:0:261418:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.341246:0:261418:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.341248:0:261418:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.341249:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21238b8 00800000:00000002:0.0:1788572473.341251:0:261418:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-073' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.341254:0:261418:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.341256:0:261418:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.341258:0:261418:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.341259:0:261418:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.341260:0:261418:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.341262:0:261418:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.341263:0:261418:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.341265:0:261418:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.341267:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123850 00800000:00000002:0.0:1788572473.341269:0:261418:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-073' -> mds #0 00000002:00000001:0.0:1788572473.341271:0:261418:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.341272:0:261418:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2123788 00000002:00010000:0.0:1788572473.341274:0:261418:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-073,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.341277:0:261418:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.341279:0:261418:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.341285:0:261418:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572473.341289:0:261418:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.341295:0:261418:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.341299:0:261418:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.341302:0:261418:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.341303:0:261418:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.341304:0:261418:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.341307:0:261418:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.341310:0:261418:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.341312:0:261418:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.341320:0:261418:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572473.341328:0:261418:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341333:0:261418:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.341335:0:261418:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341336:0:261418:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.341337:0:261418:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.341340:0:261418:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.341347:0:261418:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00010000:00000001:0.0:1788572473.341351:0:261418:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.341353:0:261418:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.341356:0:261418:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.341358:0:261418:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.341361:0:261418:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.341364:0:261418:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c37013fc600. 00000020:00000001:0.0:1788572473.341367:0:261418:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.341370:0:261418:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c37013fc600 with handle 0x14415860931686e5 to hash 00000020:00000001:0.0:1788572473.341372:0:261418:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.341374:0:261418:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634358790931968 : -109714918619648 : ffff9c37013fc600) 00010000:00000001:0.0:1788572473.341377:0:261418:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634358790931968 : -109714918619648 : ffff9c37013fc600) 00010000:00000001:0.0:1788572473.341379:0:261418:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.341381:0:261418:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.341384:0:261418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc600/0x14415860931686e5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261418 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.341392:0:261418:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc600/0x14415860931686e5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261418 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.341402:0:261418:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37013fc600/0x14415860931686e5 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261418 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.341407:0:261418:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.341409:0:261418:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.341414:0:261418:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb56c0. 00000100:00000001:0.0:1788572473.341416:0:261418:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288714944 : -109714420836672 : ffff9c371eeb56c0) 00000100:00000001:0.0:1788572473.341419:0:261418:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.341420:0:261418:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788572473.341424:0:261418:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.341428:0:261418:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.341430:0:261418:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.341432:0:261418:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.341444:0:261418:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809951104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.341452:0:261418:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.341455:0:261418:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.341458:0:261418:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.341461:0:261418:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.341463:0:261418:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.341466:0:261418:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261418:1875440809951104:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.341471:0:261418:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.341476:0:261418:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.341478:0:261418:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.341481:0:261418:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.341486:0:261418:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef800. 02000000:00000001:0.0:1788572473.341488:0:261418:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.341495:0:261418:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.341499:0:261418:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c80b0. 00000400:00000010:0.0:1788572473.341505:0:261418:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572473.341513:0:261418:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809951104, portal 10 00000100:00000001:0.0:1788572473.341515:0:261418:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.341516:0:261418:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572473.341522:0:261418:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5f80 x1875440809951104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.341530:0:261418:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.341533:0:261418:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.341535:0:261418:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572473.341538:0:261418:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809951104, offset 0 00000400:00000200:0.0:1788572473.341544:0:261418:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.341552:0:261418:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.341560:0:261418:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.341565:0:261418:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.341569:0:261418:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.341572:0:261418:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.341575:0:261418:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.341580:0:261418:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539810:539810:256:4294967295] 192.168.202.148@tcp LPNI seq info [539775:539775:8:4294967295] 00000400:00000200:0.0:1788572473.341584:0:261418:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.341594:0:261418:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.341600:0:261418:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.341605:0:261418:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572473.341613:0:261418:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.341621:0:261418:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.341626:0:261418:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.341677:0:261418:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.341681:0:261418:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15005 usecs (16 jiffies) 00000400:00000200:0.0:1788572473.341685:0:261418:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.341688:0:261418:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612362 00000100:00000001:0.0:1788572473.341691:0:261418:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.341693:0:261418:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612362 00000100:00000001:0.0:1788572473.341695:0:261418:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.341697:0:261418:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.341699:0:261418:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.341702:0:261418:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.341703:0:261418:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.341706:0:261418:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb56c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.341708:0:261418:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.341710:0:261418:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.363184:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.363201:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.363206:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.363209:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.363219:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.363227:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.363236:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.363241:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.363246:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.363252:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417463f80 00000400:00000200:0.0:1788572473.363257:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.363263:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529fb9 [1] + 224 00000800:00000001:0.0:1788572473.363270:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.363326:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.363330:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.363334:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.363343:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.363352:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5f80 x1875440809951104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.363364:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b5f80 x1875440809951104/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.363387:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.363394:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.363397:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.372268:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.372281:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.372284:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.372286:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.372295:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.372303:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.372312:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.372317:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.372321:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.372325:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464000 00000400:00000200:0.0:1788572473.372329:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.372335:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529fc9 [1] + 224 00000800:00000001:0.0:1788572473.372342:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.372348:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.372351:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.372355:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.372362:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.372371:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70a80 x1875440809951232/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.372381:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70a80 x1875440809951232/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.372398:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.372406:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.372409:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.411289:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.411299:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.411302:0:261420:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.411306:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bb20 00000080:00200000:0.0:1788572473.411311:0:261420:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.411319:0:261420:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572473.411325:0:261420:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.411330:0:261420:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.411335:0:261420:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.411337:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211ba40 00800000:00000002:0.0:1788572473.411340:0:261420:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.411343:0:261420:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.411346:0:261420:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.411349:0:261420:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.411352:0:261420:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.411355:0:261420:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.411357:0:261420:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.411358:0:261420:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.411361:0:261420:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.411364:0:261420:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.411370:0:261420:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.411372:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b910 00000002:00010000:0.0:1788572473.411374:0:261420:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.411378:0:261420:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.411380:0:261420:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.411383:0:261420:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.411391:0:261420:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.411394:0:261420:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.411400:0:261420:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.411405:0:261420:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.411409:0:261420:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 246 lru_type: 1 02000000:00000001:0.0:1788572473.411421:0:261420:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.411428:0:261420:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.411431:0:261420:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 246 lru_type: 1 00000002:00000001:0.0:1788572473.411438:0:261420:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.411441:0:261420:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.411443:0:261420:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.411445:0:261420:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.411447:0:261420:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.411449:0:261420:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.411452:0:261420:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.411455:0:261420:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.411458:0:261420:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572473.411465:0:261420:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.411467:0:261420:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.411469:0:261420:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.411472:0:261420:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.411475:0:261420:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.411478:0:261420:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.411479:0:261420:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.411482:0:261420:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.411485:0:261420:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.411487:0:261420:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.411490:0:261420:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.411492:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.411494:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.411495:0:261420:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.411497:0:261420:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.411500:0:261420:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc211bb58 00010000:00000001:0.0:1788572473.411502:0:261420:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.411503:0:261420:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.411504:0:261420:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.411506:0:261420:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.411507:0:261420:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.411509:0:261420:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.411512:0:261420:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 246 lru_type: 1 00010000:00000001:0.0:1788572473.411519:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.411519:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.411521:0:261420:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.411523:0:261420:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.411525:0:261420:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 247 lru_type: 1 00010000:00000001:0.0:1788572473.411533:0:261420:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.411535:0:261420:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.411537:0:261420:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.411541:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.411542:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.411544:0:261420:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411547:0:261420:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.411549:0:261420:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411551:0:261420:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411554:0:261420:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411556:0:261420:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.411557:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.411558:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.411561:0:261420:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.411563:0:261420:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.411564:0:261420:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.411569:0:261420:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.411573:0:261420:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.411575:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bae8 00000080:00002000:0.0:1788572473.411577:0:261420:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 73 00000080:00000001:0.0:1788572473.411579:0:261420:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.411581:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bb30 00000080:00200000:0.0:1788572473.411582:0:261420:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572473.411587:0:261420:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.411592:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.411593:0:261420:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.411596:0:261420:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.411605:0:261420:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310fba0. 00000020:00000001:0.0:1788572473.411608:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bae8 00000080:00200000:0.0:1788572473.411610:0:261420:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-074, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.411613:0:261420:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.411614:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b998 00000080:00200000:0.0:1788572473.411616:0:261420:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-074, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.411624:0:261420:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572473.411626:0:261420:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.411630:0:261420:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.411632:0:261420:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.411633:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b8b8 00800000:00000002:0.0:1788572473.411657:0:261420:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-074' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.411660:0:261420:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.411664:0:261420:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.411665:0:261420:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.411667:0:261420:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.411668:0:261420:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.411670:0:261420:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.411672:0:261420:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.411674:0:261420:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.411677:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b850 00800000:00000002:0.0:1788572473.411679:0:261420:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-074' -> mds #0 00000002:00000001:0.0:1788572473.411682:0:261420:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.411683:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b788 00000002:00010000:0.0:1788572473.411685:0:261420:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-074,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.411689:0:261420:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.411692:0:261420:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.411697:0:261420:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572473.411702:0:261420:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.411709:0:261420:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.411715:0:261420:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.411718:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.411719:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.411721:0:261420:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.411724:0:261420:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.411726:0:261420:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.411729:0:261420:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.411736:0:261420:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572473.411745:0:261420:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411752:0:261420:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.411754:0:261420:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411755:0:261420:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.411757:0:261420:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.411758:0:261420:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.411767:0:261420:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00010000:00000001:0.0:1788572473.411772:0:261420:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.411774:0:261420:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.411778:0:261420:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.411780:0:261420:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.411783:0:261420:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.411789:0:261420:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30c00. 00000020:00000001:0.0:1788572473.411792:0:261420:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.411795:0:261420:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30c00 with handle 0x14415860931686f3 to hash 00000020:00000001:0.0:1788572473.411797:0:261420:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.411799:0:261420:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572473.411802:0:261420:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572473.411804:0:261420:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.411806:0:261420:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.411809:0:261420:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x14415860931686f3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261420 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.411820:0:261420:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x14415860931686f3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261420 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.411831:0:261420:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x14415860931686f3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261420 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.411837:0:261420:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.411838:0:261420:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.411845:0:261420:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb53c0. 00000100:00000001:0.0:1788572473.411848:0:261420:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288714176 : -109714420837440 : ffff9c371eeb53c0) 00000100:00000001:0.0:1788572473.411850:0:261420:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.411851:0:261420:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00000100:00000001:0.0:1788572473.411856:0:261420:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.411860:0:261420:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.411862:0:261420:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.411863:0:261420:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.411874:0:261420:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5880 x1875440809951360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.411882:0:261420:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.411884:0:261420:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.411888:0:261420:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.411903:0:261420:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.411906:0:261420:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.411909:0:261420:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5880 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261420:1875440809951360:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.411914:0:261420:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.411919:0:261420:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.411921:0:261420:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.411923:0:261420:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.411929:0:261420:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e8800. 02000000:00000001:0.0:1788572473.411931:0:261420:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.411939:0:261420:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.411943:0:261420:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c86e0. 00000400:00000010:0.0:1788572473.411951:0:261420:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572473.411958:0:261420:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809951360, portal 10 00000100:00000001:0.0:1788572473.411960:0:261420:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.411961:0:261420:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00000100:00000040:0.0:1788572473.411967:0:261420:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5880 x1875440809951360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.411973:0:261420:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.411975:0:261420:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.411977:0:261420:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788572473.411980:0:261420:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809951360, offset 0 00000400:00000200:0.0:1788572473.411986:0:261420:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.411995:0:261420:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.412003:0:261420:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.412008:0:261420:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.412013:0:261420:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.412017:0:261420:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.412020:0:261420:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.412024:0:261420:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539812:539812:256:4294967295] 192.168.202.148@tcp LPNI seq info [539777:539777:8:4294967295] 00000400:00000200:0.0:1788572473.412028:0:261420:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.412038:0:261420:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.412044:0:261420:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.412049:0:261420:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572473.412056:0:261420:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.412063:0:261420:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.412067:0:261420:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.412091:0:261420:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.412096:0:261420:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15010 usecs (16 jiffies) 00000400:00000200:0.0:1788572473.412100:0:261420:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.412103:0:261420:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612423 00000100:00000001:0.0:1788572473.412106:0:261420:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.412109:0:261420:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612423 00000100:00000001:0.0:1788572473.412111:0:261420:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.412114:0:261420:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.412117:0:261420:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.412119:0:261420:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.412121:0:261420:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.412123:0:261420:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb53c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.412126:0:261420:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.412129:0:261420:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.433961:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.433982:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.433986:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.433990:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.433999:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.434007:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.434019:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.434024:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.434030:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.434034:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464080 00000400:00000200:0.0:1788572473.434040:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.434046:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529fd1 [1] + 224 00000800:00000001:0.0:1788572473.434054:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.434063:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.434066:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.434071:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.434078:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.434087:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5880 x1875440809951360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.434098:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b5880 x1875440809951360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.434124:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.434132:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.434135:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.439083:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.439089:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.447077:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.447101:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.447105:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.447108:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.447118:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.447125:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.447136:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.447141:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.447146:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.447150:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464100 00000400:00000200:0.0:1788572473.447156:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.447162:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x529fe1 [1] + 224 00000800:00000001:0.0:1788572473.447171:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.447185:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.447188:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.447193:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.447201:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.447210:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4e00 x1875440809951488/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.447221:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c4e00 x1875440809951488/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.447252:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.447261:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.447264:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572473.449992:0:261420:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450002:0:261420:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.450010:0:261420:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572473.450015:0:261420:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4d5f8 -> ffff9c3720b4d5f8 00000020:00000001:0.0:1788572473.450018:0:261420:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4d5f8 -> ffff9c3720b4d5f8 00000020:00000001:0.0:1788572473.450020:0:261420:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572473.450025:0:261420:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572473.450029:0:261420:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.450031:0:261420:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.450032:0:261420:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.450033:0:261420:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.450036:0:261420:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdde:11031 on OST[0]: s=0 (1) m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.450041:0:261420:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.450043:0:261420:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.450046:0:261420:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450048:0:261420:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450050:0:261420:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4d5f8 -> ffff9c3720b4d5f8 00000080:00200000:0.0:1788572473.450053:0:261420:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdde:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572473.450056:0:261420:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572473.450058:0:261420:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572473.450060:0:261420:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572473.450063:0:261420:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.450065:0:261420:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572473.450068:0:261420:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572473.450072:0:261420:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.450074:0:261420:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314391128 : -109714395160488 : ffff9c3720732058) 00000020:00000001:0.0:1788572473.450076:0:261420:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.450077:0:261420:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192a730 (ffff9c3707076ea8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572473.450082:0:261420:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572473.450083:0:261420:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572473.450085:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.450087:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.450089:0:261420:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572473.450091:0:261420:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.450092:0:261420:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.450093:0:261420:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.450094:0:261420:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.450095:0:261420:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572473.450098:0:261420:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572473.450099:0:261420:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.450100:0:261420:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192a730 (ffff9c37070767e8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572473.450103:0:261420:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313b8f0. 00000020:00000001:0.0:1788572473.450108:0:261420:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572473.450111:0:261420:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192a700. 00020000:00000001:0.0:1788572473.450114:0:261420:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.450116:0:261420:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.450117:0:261420:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572473.450118:0:261420:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450121:0:261420:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572473.450124:0:261420:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572473.450125:0:261420:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572473.450127:0:261420:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572473.450128:0:261420:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.450130:0:261420:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076ea0 00020000:00000001:0.0:1788572473.450134:0:261420:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572473.450136:0:261420:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572473.450139:0:261420:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572473.450142:0:261420:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdde:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572473.450146:0:261420:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.450148:0:261420:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.450150:0:261420:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572473.450152:0:261420:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572473.450153:0:261420:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.450154:0:261420:0:(cl_object.c:917:cl_env_put()) 1@ffff9c37070767e0 00000001:00000001:0.0:1788572473.450156:0:261420:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450166:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bd08 00000080:00000008:0.0:1788572473.450168:0:261420:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-074:[0x200002b17:0xdde:0x0](ffff9c372f2d7448), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.450174:0:261420:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.450176:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bda0 00000080:00002000:0.0:1788572473.450178:0:261420:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-074 (ffff9c370804fe58, parent ffff9c3719d93af8, inode ffff9c372f2d7448) hashed, 00000080:00000001:0.0:1788572473.450181:0:261420:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.450210:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.450214:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.450216:0:261420:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.450218:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bbc0 00000080:00200000:0.0:1788572473.450220:0:261420:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.450226:0:261420:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572473.450232:0:261420:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.450236:0:261420:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.450241:0:261420:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.450243:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bae0 00800000:00000002:0.0:1788572473.450245:0:261420:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.450250:0:261420:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.450252:0:261420:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.450256:0:261420:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.450258:0:261420:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.450261:0:261420:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.450263:0:261420:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.450265:0:261420:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.450268:0:261420:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.450271:0:261420:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.450276:0:261420:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.450278:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211b9b0 00000002:00010000:0.0:1788572473.450280:0:261420:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.450283:0:261420:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.450285:0:261420:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.450288:0:261420:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.450296:0:261420:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.450298:0:261420:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.450303:0:261420:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.450307:0:261420:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.450311:0:261420:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 247 lru_type: 1 02000000:00000001:0.0:1788572473.450323:0:261420:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.450328:0:261420:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.450331:0:261420:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 247 lru_type: 1 00000002:00000001:0.0:1788572473.450338:0:261420:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.450341:0:261420:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.450343:0:261420:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.450345:0:261420:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.450347:0:261420:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.450349:0:261420:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.450352:0:261420:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.450356:0:261420:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.450359:0:261420:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6600. 00000080:00000001:0.0:1788572473.450367:0:261420:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.450368:0:261420:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.450371:0:261420:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.450374:0:261420:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.450378:0:261420:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.450380:0:261420:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.450382:0:261420:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.450383:0:261420:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.450386:0:261420:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.450389:0:261420:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.450392:0:261420:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.450394:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.450395:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.450397:0:261420:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.450399:0:261420:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.450401:0:261420:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc211bbf8 00010000:00000001:0.0:1788572473.450403:0:261420:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.450404:0:261420:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.450406:0:261420:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.450408:0:261420:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.450410:0:261420:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.450412:0:261420:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.450415:0:261420:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 247 lru_type: 1 00010000:00000001:0.0:1788572473.450422:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.450423:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.450424:0:261420:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.450426:0:261420:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.450429:0:261420:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 248 lru_type: 1 00010000:00000001:0.0:1788572473.450441:0:261420:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.450443:0:261420:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.450445:0:261420:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.450449:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.450450:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.450452:0:261420:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.450454:0:261420:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.450456:0:261420:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.450459:0:261420:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.450461:0:261420:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.450463:0:261420:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.450464:0:261420:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.450465:0:261420:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.450467:0:261420:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.450469:0:261420:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.450471:0:261420:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.450475:0:261420:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.450478:0:261420:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.450480:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bb88 00000080:00002000:0.0:1788572473.450482:0:261420:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 74 00000080:00000001:0.0:1788572473.450484:0:261420:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450486:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bbd0 00000080:00200000:0.0:1788572473.450487:0:261420:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572473.450492:0:261420:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.450495:0:261420:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.450497:0:261420:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.450499:0:261420:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.450501:0:261420:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.450502:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bbe8 00000080:00002000:0.0:1788572473.450504:0:261420:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-074(ffff9c370804fe58) flags 0x48000e refc 0 00000080:00000001:0.0:1788572473.450506:0:261420:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.450508:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211bc30 00000080:00200000:0.0:1788572473.450509:0:261420:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-074, flags=9 00000080:00000001:0.0:1788572473.450519:0:261420:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572473.450523:0:261420:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572473.450525:0:261420:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.450526:0:261420:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc211be40 00000080:00002000:0.0:1788572473.450528:0:261420:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-074 (ffff9c370804fe58, parent ffff9c3719d93af8, inode ffff9c372f2d7448) hashed, 00000080:00000001:0.0:1788572473.450531:0:261420:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572473.496924:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.496932:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572473.496937:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.496943:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.496950:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788572473.496952:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788572473.496957:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.496963:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4e00 x1875440809951616/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.496973:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.496976:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4e00 x1875440809951616/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.496982:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.496984:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.506282:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.506302:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.506307:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.506309:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.506320:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.506329:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.506338:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.506343:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.506349:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.506353:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464180 00000400:00000200:0.0:1788572473.506359:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.506364:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x529ff1 [1] + 224 00000800:00000001:0.0:1788572473.506372:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.506429:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.506434:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.506439:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.506448:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.506457:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4e00 x1875440809951616/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.506468:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4e00 x1875440809951616/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.506488:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.506496:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.506499:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.508792:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.508797:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572473.508802:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.508807:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.508812:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1220 00000400:00000010:0.0:1788572473.508814:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1220. 00000100:00000001:0.0:1788572473.508817:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.508822:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a680 x1875440809951744/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.508836:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.508839:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440809951744/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.508845:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.508847:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.512158:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.512176:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.512181:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.512184:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.512192:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.512200:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.512210:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.512215:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.512220:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.512224:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464200 00000400:00000200:0.0:1788572473.512229:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.512236:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a001 [1] + 224 00000800:00000001:0.0:1788572473.512244:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.512251:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.512255:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.512260:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.512268:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.512276:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809951744/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.512287:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6a680 x1875440809951744/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.512309:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.512317:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.512320:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000002:0.0:1788572473.568550:0:261424:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.568567:0:261424:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.568570:0:261424:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.568578:0:261424:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.568595:0:261424:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f520. 00000020:00000001:0.0:1788572473.568602:0:261424:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bae8 00000080:00200000:0.0:1788572473.568605:0:261424:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-076, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.568608:0:261424:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.568611:0:261424:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215b998 00000080:00200000:0.0:1788572473.568613:0:261424:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-076, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.568622:0:261424:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6600. 00800000:00000002:0.0:1788572473.568628:0:261424:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.568633:0:261424:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.568725:0:261424:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.568728:0:261424:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215b8b8 00800000:00000002:0.0:1788572473.568730:0:261424:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-076' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.568733:0:261424:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.568737:0:261424:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.568740:0:261424:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.568742:0:261424:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.568745:0:261424:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.568747:0:261424:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.568749:0:261424:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.568751:0:261424:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.568754:0:261424:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215b850 00800000:00000002:0.0:1788572473.568755:0:261424:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-076' -> mds #0 00000002:00000001:0.0:1788572473.568761:0:261424:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.568763:0:261424:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215b788 00000002:00010000:0.0:1788572473.568764:0:261424:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-076,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.568768:0:261424:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.568770:0:261424:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.568777:0:261424:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572473.568781:0:261424:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.568787:0:261424:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.568792:0:261424:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.568799:0:261424:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.568800:0:261424:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.568802:0:261424:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.568804:0:261424:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.568806:0:261424:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.568810:0:261424:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.568818:0:261424:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572473.568827:0:261424:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.568832:0:261424:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.568834:0:261424:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.568836:0:261424:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.568837:0:261424:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.568839:0:261424:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.568845:0:261424:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00010000:00000001:0.0:1788572473.568849:0:261424:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.568851:0:261424:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.568857:0:261424:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.568860:0:261424:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.568863:0:261424:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.568867:0:261424:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31800. 00000020:00000001:0.0:1788572473.568870:0:261424:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.568873:0:261424:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31800 with handle 0x144158609316870f to hash 00000020:00000001:0.0:1788572473.568875:0:261424:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.568876:0:261424:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572473.568880:0:261424:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572473.568883:0:261424:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.568885:0:261424:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.568889:0:261424:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x144158609316870f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261424 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.568908:0:261424:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x144158609316870f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261424 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.568918:0:261424:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x144158609316870f lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261424 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.568923:0:261424:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.568925:0:261424:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.568931:0:261424:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5300. 00000100:00000001:0.0:1788572473.568933:0:261424:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288713984 : -109714420837632 : ffff9c371eeb5300) 00000100:00000001:0.0:1788572473.568935:0:261424:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.568937:0:261424:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788572473.568941:0:261424:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.568944:0:261424:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.568947:0:261424:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.568948:0:261424:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.568958:0:261424:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440809951872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.568966:0:261424:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.568968:0:261424:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.568971:0:261424:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.568974:0:261424:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.568977:0:261424:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.568980:0:261424:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261424:1875440809951872:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.568984:0:261424:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.568991:0:261424:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.568994:0:261424:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.568996:0:261424:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.569002:0:261424:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e8800. 02000000:00000001:0.0:1788572473.569004:0:261424:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.569012:0:261424:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.569016:0:261424:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8058. 00000400:00000010:0.0:1788572473.569023:0:261424:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572473.569029:0:261424:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809951872, portal 10 00000100:00000001:0.0:1788572473.569031:0:261424:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.569032:0:261424:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000040:0.0:1788572473.569038:0:261424:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4a80 x1875440809951872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.569044:0:261424:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.569047:0:261424:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.569050:0:261424:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788572473.569052:0:261424:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809951872, offset 0 00000400:00000200:0.0:1788572473.569059:0:261424:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.569067:0:261424:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.569074:0:261424:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.569078:0:261424:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.569083:0:261424:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.569086:0:261424:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.569089:0:261424:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.569093:0:261424:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539816:539816:256:4294967295] 192.168.202.148@tcp LPNI seq info [539781:539781:8:4294967295] 00000400:00000200:0.0:1788572473.569097:0:261424:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.569236:0:261424:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.569243:0:261424:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.569249:0:261424:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572473.569256:0:261424:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.569263:0:261424:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.569267:0:261424:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.569319:0:261424:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.569324:0:261424:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15013 usecs (16 jiffies) 00000400:00000200:0.0:1788572473.569328:0:261424:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.569331:0:261424:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612591 00000100:00000001:0.0:1788572473.569334:0:261424:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.569337:0:261424:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612591 00000100:00000001:0.0:1788572473.569339:0:261424:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.569341:0:261424:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.569344:0:261424:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.569346:0:261424:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.569348:0:261424:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.569351:0:261424:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5300 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.569354:0:261424:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.569357:0:261424:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.577908:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.577925:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.577930:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.577933:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.577941:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.577950:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.577959:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.577964:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.577969:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.577974:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464280 00000400:00000200:0.0:1788572473.577979:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.577985:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a009 [1] + 224 00000800:00000001:0.0:1788572473.577993:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.578052:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.578057:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.578063:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.578073:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.578085:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4a80 x1875440809951872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.578097:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4a80 x1875440809951872/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.578120:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.578129:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.578133:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.584111:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.584128:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.584133:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.584136:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.584144:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.584152:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.584161:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.584166:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.584171:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.584175:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464300 00000400:00000200:0.0:1788572473.584180:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.584186:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a019 [1] + 224 00000800:00000001:0.0:1788572473.584194:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.584202:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.584205:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.584210:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.584219:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.584229:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809952000/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.584239:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6a680 x1875440809952000/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.587662:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.587687:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.587697:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.632092:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.632117:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.632122:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.632125:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.632134:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.632142:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.632150:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.632154:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.632159:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.632163:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464380 00000400:00000200:0.0:1788572473.632168:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.632174:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a021 [1] + 224 00000800:00000001:0.0:1788572473.632182:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.632188:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.632192:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.632196:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.632204:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.632212:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7800 x1875440809952128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.632222:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c7800 x1875440809952128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.632242:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.632250:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.632254:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.634097:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.634108:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.634110:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.634117:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7800 x1875440809952256/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.634130:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.634132:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.634135:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.634138:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.634140:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.634143:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809952256:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.634148:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.634153:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.634156:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.634158:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.634163:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572473.634165:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.634172:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.634176:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c83c8. 00000400:00000010:0.0:1788572473.634183:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572473.634189:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809952256, portal 4 00000100:00000001:0.0:1788572473.634191:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.634193:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000040:0.0:1788572473.634199:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c7800 x1875440809952256/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.634207:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.634209:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.634212:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31220. 00000100:00000200:0.0:1788572473.634216:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809952256, offset 0 00000400:00000200:0.0:1788572473.634222:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.634231:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.634238:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.634243:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.634248:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.634252:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.634254:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.634259:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539819:539819:256:4294967295] 192.168.202.148@tcp LPNI seq info [539784:539784:8:4294967295] 00000400:00000200:0.0:1788572473.634264:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.634274:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.634280:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.634287:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572473.634294:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.634303:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.634309:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.634330:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.634336:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 176521 usecs (177 jiffies) 00000400:00000200:0.0:1788572473.634340:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.634342:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612817 00000100:00000001:0.0:1788572473.634346:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.634348:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612817 00000100:00000001:0.0:1788572473.634352:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.634354:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.634357:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.634360:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.634363:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.634369:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.634372:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572473.634374:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.634375:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.634377:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.634379:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.634388:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.634390:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.638055:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.638075:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.638081:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.638083:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.638093:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.638113:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.638125:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.638130:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.638135:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.638139:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464400 00000400:00000200:0.0:1788572473.638144:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.638151:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a031 [1] + 224 00000800:00000001:0.0:1788572473.638158:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.638167:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.638170:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.638174:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.638183:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.638193:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7800 x1875440809952256/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.638204:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c7800 x1875440809952256/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.638227:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.638235:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.638239:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.661090:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.661095:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572473.661151:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.661157:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.661163:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572473.661166:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572473.661170:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.661174:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7800 x1875440809952384/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.661184:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.661188:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7800 x1875440809952384/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.661199:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.661201:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.685586:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.706490:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.706508:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.706512:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.706515:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.706523:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.706532:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.706540:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.706545:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.706551:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.706555:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464500 00000400:00000200:0.0:1788572473.706560:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.706566:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a051 [1] + 224 00000800:00000001:0.0:1788572473.706574:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.706581:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.706585:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.706588:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.706596:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.706605:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70a80 x1875440809952512/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.706615:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70a80 x1875440809952512/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.706732:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.706741:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.706745:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.731551:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.731562:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572473.731570:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.731579:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.731589:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf13b8 00000400:00000010:0.0:1788572473.731592:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf13b8. 00000100:00000001:0.0:1788572473.731598:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.731608:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809952640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.731620:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.731624:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6aa00 x1875440809952640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.731632:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.731659:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.747042:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.747069:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.747076:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.747082:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.747090:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.747101:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.747119:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.747123:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.747128:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.747132:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464580 00000400:00000200:0.0:1788572473.747137:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.747143:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a059 [1] + 224 00000800:00000001:0.0:1788572473.747151:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.747165:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.747171:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.747175:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.747187:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.747198:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809952640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.747215:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6aa00 x1875440809952640/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.747242:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.747256:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.747260:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.749274:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.749279:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572473.749283:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.749288:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.749293:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1440 00000400:00000010:0.0:1788572473.749295:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1440. 00000100:00000001:0.0:1788572473.749299:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.749303:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809952768/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.749312:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.749316:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440809952768/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.749322:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.749323:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.754993:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.755013:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.755018:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.755020:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.755030:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.755037:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.755046:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.755051:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.755056:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.755061:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464600 00000400:00000200:0.0:1788572473.755065:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.755072:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a061 [1] + 224 00000800:00000001:0.0:1788572473.755079:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.755086:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.755089:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.755094:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.755101:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.755124:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809952768/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.755136:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6aa00 x1875440809952768/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.755155:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.755163:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.755166:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.798152:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.798162:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572473.798169:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.798177:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.798185:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1990 00000400:00000010:0.0:1788572473.798188:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1990. 00000100:00000001:0.0:1788572473.798194:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.798201:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809952896/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.798212:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.798216:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71500 x1875440809952896/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.798223:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.798224:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.811984:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.812002:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.812006:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.812009:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.812017:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.812026:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.812034:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.812039:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.812044:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.812048:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464680 00000400:00000200:0.0:1788572473.812054:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.812061:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a069 [1] + 224 00000800:00000001:0.0:1788572473.812069:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.812075:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.812079:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.812084:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.812092:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.812102:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809952896/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.812177:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec71500 x1875440809952896/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.812198:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.812208:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.812212:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572473.814792:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00200008:0.0:1788572473.814800:0:261432:0:(file.c:6296:ll_getattr_dentry()) START file f434.sanity-080:[0x200002b17:0xde4:0x0](ffff9c372f2d3a88), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.814808:0:261432:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.814810:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc80 00000080:00200000:0.0:1788572473.814812:0:261432:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xde4:0x0](ffff9c372f2d3a88),name=f434.sanity-080 00000080:00000010:0.0:1788572473.814822:0:261432:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572473.814828:0:261432:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.814834:0:261432:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.814839:0:261432:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.814841:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bba0 00800000:00000002:0.0:1788572473.814843:0:261432:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xde4:0x0] 'f434.sanity-080' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.814847:0:261432:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.814851:0:261432:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.814854:0:261432:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.814856:0:261432:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.814859:0:261432:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.814861:0:261432:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.814864:0:261432:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.814866:0:261432:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.814869:0:261432:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.814871:0:261432:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.814872:0:261432:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.814873:0:261432:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.814874:0:261432:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.814876:0:261432:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xde4:0x0] 00800000:00000001:0.0:1788572473.814877:0:261432:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.814880:0:261432:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xde4:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.814885:0:261432:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.814886:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207ba70 00000002:00010000:0.0:1788572473.814888:0:261432:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xde4:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.814901:0:261432:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.814904:0:261432:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.814906:0:261432:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.814912:0:261432:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3705277000 count: 3 00010000:00000001:0.0:1788572473.814915:0:261432:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.814919:0:261432:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.814923:0:261432:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.814927:0:261432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee8d800/0x1441586093168747 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xde4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2959 expref: -99 pid: 261432 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572473.814936:0:261432:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.814941:0:261432:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.814944:0:261432:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee8d800/0x1441586093168747 lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xde4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2959 expref: -99 pid: 261432 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572473.814951:0:261432:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.814953:0:261432:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.814955:0:261432:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.814957:0:261432:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.814959:0:261432:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.814962:0:261432:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572473.814964:0:261432:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.814967:0:261432:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572473.814970:0:261432:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572473.814978:0:261432:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.814980:0:261432:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.814982:0:261432:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xde4:0x0](ffff9c372f2d3a88) 00000080:00010000:0.0:1788572473.814986:0:261432:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xde4:0x0](ffff9c372f2d3a88) for lock 0x1441586093168747 00800000:00000001:0.0:1788572473.814990:0:261432:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.814991:0:261432:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.814993:0:261432:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.814995:0:261432:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.814997:0:261432:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372ee8d800 refcount=3 00000020:00000001:0.0:1788572473.815000:0:261432:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359556986880 : -109714152564736 : ffff9c372ee8d800) 00010000:00000001:0.0:1788572473.815002:0:261432:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359556986880 : -109714152564736 : ffff9c372ee8d800) 00010000:00000001:0.0:1788572473.815005:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.815006:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.815008:0:261432:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.815009:0:261432:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.815012:0:261432:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168747 from it ffffb95bc207bcb8 00010000:00000001:0.0:1788572473.815014:0:261432:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.815015:0:261432:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.815016:0:261432:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372ee8d800 refcount=3 00000020:00000001:0.0:1788572473.815018:0:261432:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359556986880 : -109714152564736 : ffff9c372ee8d800) 00010000:00000001:0.0:1788572473.815021:0:261432:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359556986880 : -109714152564736 : ffff9c372ee8d800) 00010000:00000001:0.0:1788572473.815023:0:261432:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.815026:0:261432:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee8d800/0x1441586093168747 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xde4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2959 expref: -99 pid: 261432 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572473.815033:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.815034:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.815036:0:261432:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.815038:0:261432:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.815041:0:261432:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372ee8d800/0x1441586093168747 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xde4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2959 expref: -99 pid: 261432 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572473.815049:0:261432:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.815052:0:261432:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.815054:0:261432:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.815057:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.815059:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.815060:0:261432:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815063:0:261432:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.815066:0:261432:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815068:0:261432:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815071:0:261432:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815072:0:261432:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.815074:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.815075:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.815077:0:261432:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.815079:0:261432:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.815082:0:261432:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572473.815085:0:261432:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572473.815087:0:261432:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572473.815089:0:261432:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572473.815090:0:261432:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.815092:0:261432:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572473.815095:0:261432:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572473.815097:0:261432:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.815100:0:261432:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887901160 : -109714821650456 : ffff9c37070767e8) 00000020:00001000:0.0:1788572473.815102:0:261432:0:(cl_object.c:823:cl_env_get()) 1@ffff9c37070767e0 00000020:00000001:0.0:1788572473.815172:0:261432:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4daa8 -> ffff9c3720b4daa8 00000020:00000001:0.0:1788572473.815176:0:261432:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572473.815178:0:261432:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572473.815180:0:261432:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xde4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572473.815183:0:261432:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.815185:0:261432:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572473.815187:0:261432:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.815189:0:261432:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.815191:0:261432:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.815195:0:261432:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xde4:0x0]io ffff9c3731325c20 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572473.815198:0:261432:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c371fcbc9b0) owner 0000000000000000/ffff9c372d1c4f80 00020000:00000001:0.0:1788572473.815201:0:261432:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572473.815203:0:261432:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572473.815205:0:261432:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572473.815206:0:261432:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815209:0:261432:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572473.815211:0:261432:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572473.815213:0:261432:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815214:0:261432:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.815215:0:261432:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572473.815217:0:261432:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572473.815218:0:261432:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c371fcbc9b0) owner 0000000000000000/ffff9c372d1c4f80 00000020:00000001:0.0:1788572473.815220:0:261432:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572473.815222:0:261432:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572473.815223:0:261432:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xde4:0x0] 00000020:00000001:0.0:1788572473.815226:0:261432:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572473.815228:0:261432:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572473.815230:0:261432:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572473.815232:0:261432:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572473.815236:0:261432:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572473.815242:0:261432:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676100. 00020000:00000002:0.0:1788572473.815245:0:261432:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572473.815247:0:261432:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572473.815249:0:261432:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.815250:0:261432:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572473.815251:0:261432:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572473.815253:0:261432:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887902888 : -109714821648728 : ffff9c3707076ea8) 00000020:00001000:0.0:1788572473.815255:0:261432:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3707076ea0 00000020:00000001:0.0:1788572473.815257:0:261432:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4d9e0 -> ffff9c3720b4daa8 00000020:00000001:0.0:1788572473.815259:0:261432:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572473.815261:0:261432:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572473.815262:0:261432:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572473.815263:0:261432:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815265:0:261432:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815267:0:261432:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314391128 : -109714395160488 : ffff9c3720732058) 00000020:00000001:0.0:1788572473.815268:0:261432:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572473.815272:0:261432:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6d10. 00000008:00010000:0.0:1788572473.815275:0:261432:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572473.815277:0:261432:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572473.815278:0:261432:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572473.815280:0:261432:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676130, osc lock ffff9c37030a6d10, flags 0x400001002 00000020:00000001:0.0:1788572473.815282:0:261432:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815284:0:261432:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815286:0:261432:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725056 : -109714177826560 : ffff9c372d676100) 00000020:00000001:0.0:1788572473.815289:0:261432:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572473.815290:0:261432:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572473.815292:0:261432:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815293:0:261432:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815296:0:261432:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572473.815298:0:261432:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572473.815299:0:261432:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572473.815301:0:261432:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572473.815303:0:261432:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.815304:0:261432:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314391128 : -109714395160488 : ffff9c3720732058) 00000020:00000001:0.0:1788572473.815305:0:261432:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572473.815307:0:261432:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572473.815309:0:261432:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572473.815313:0:261432:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572473.815315:0:261432:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572473.815316:0:261432:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572473.815318:0:261432:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572473.815320:0:261432:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572473.815321:0:261432:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572473.815323:0:261432:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572473.815325:0:261432:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572473.815329:0:261432:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815332:0:261432:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572473.815336:0:261432:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572473.815340:0:261432:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572473.815344:0:261432:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.815348:0:261432:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.815350:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.815352:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.815353:0:261432:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.815356:0:261432:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.815358:0:261432:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.815362:0:261432:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.815367:0:261432:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b7000. 00000100:00000001:0.0:1788572473.815374:0:261432:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815378:0:261432:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.815380:0:261432:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815381:0:261432:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.815383:0:261432:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815385:0:261432:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.815387:0:261432:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531979648 : -109714177571968 : ffff9c372d6b4380) 00010000:00000001:0.0:1788572473.815390:0:261432:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.815392:0:261432:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572473.815398:0:261432:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3710a87f00. 00010000:00000010:0.0:1788572473.815401:0:261432:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3710a87c00. 00010000:00000001:0.0:1788572473.815405:0:261432:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.815408:0:261432:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788572473.815412:0:261432:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.815414:0:261432:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x144158609316874e to hash 00000020:00000001:0.0:1788572473.815415:0:261432:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.815417:0:261432:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000010:0.0:1788572473.815420:0:261432:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091080. 00010000:00000001:0.0:1788572473.815423:0:261432:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788572473.815426:0:261432:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.815427:0:261432:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.815431:0:261432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000055cc5573/0x144158609316874e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5283:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261432 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.815442:0:261432:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000055cc5573/0x144158609316874e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x5283:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261432 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.815452:0:261432:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.815457:0:261432:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.815461:0:261432:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.815471:0:261432:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4380 x1875440809953024/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572473.815847:0:261432:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572473.815850:0:261432:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815852:0:261432:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.815854:0:261432:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815856:0:261432:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.815858:0:261432:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572473.815860:0:261432:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572473.815861:0:261432:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572473.815862:0:261432:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000800:00000200:0.0:1788572473.815963:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.815967:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572473.815972:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.815977:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.815983:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572473.815985:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572473.815989:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.815994:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4380 x1875440809953024/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.816003:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.816006:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4380 x1875440809953024/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.816014:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.816015:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572473.818412:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.818419:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.829011:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.829029:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.829033:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.829036:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.829045:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.829053:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.829062:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.829067:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.829072:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.829077:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464700 00000400:00000200:0.0:1788572473.829082:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.829088:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a071 [1] + 224 00000800:00000001:0.0:1788572473.829095:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.829103:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.829169:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.829174:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.829183:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.829192:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4380 x1875440809953024/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.829204:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c372d6b4380 x1875440809953024/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.829226:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.829235:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.829239:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572473.829737:0:261432:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.829741:0:261432:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.829747:0:261432:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572473.829749:0:261432:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4daa8 -> ffff9c3720b4daa8 00000020:00000001:0.0:1788572473.829751:0:261432:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4daa8 -> ffff9c3720b4daa8 00000020:00000001:0.0:1788572473.829753:0:261432:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572473.829757:0:261432:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572473.829761:0:261432:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.829763:0:261432:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.829764:0:261432:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.829765:0:261432:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.829767:0:261432:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xde4:11031 on OST[0]: s=0 (1) m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.829771:0:261432:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.829772:0:261432:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.829775:0:261432:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.829777:0:261432:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.829778:0:261432:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4daa8 -> ffff9c3720b4daa8 00000080:00200000:0.0:1788572473.829780:0:261432:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xde4:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572473.829782:0:261432:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572473.829784:0:261432:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572473.829786:0:261432:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572473.829790:0:261432:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.829791:0:261432:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572473.829795:0:261432:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572473.829798:0:261432:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.829800:0:261432:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314391128 : -109714395160488 : ffff9c3720732058) 00000020:00000001:0.0:1788572473.829802:0:261432:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.829803:0:261432:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c372d676130 (ffff9c3707076ea8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572473.829806:0:261432:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572473.829807:0:261432:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572473.829808:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.829809:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.829810:0:261432:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572473.829812:0:261432:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.829813:0:261432:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.829814:0:261432:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.829815:0:261432:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.829817:0:261432:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731325bc0 (ffff9c37070767e8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572473.829820:0:261432:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572473.829821:0:261432:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.829822:0:261432:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c372d676130 (ffff9c37070767e8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572473.829825:0:261432:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c37030a6d10. 00000020:00000001:0.0:1788572473.829828:0:261432:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572473.829830:0:261432:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c372d676100. 00020000:00000001:0.0:1788572473.829836:0:261432:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.829837:0:261432:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.829838:0:261432:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572473.829840:0:261432:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.829842:0:261432:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572473.829845:0:261432:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572473.829847:0:261432:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572473.829848:0:261432:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572473.829849:0:261432:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.829851:0:261432:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076ea0 00020000:00000001:0.0:1788572473.829854:0:261432:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572473.829856:0:261432:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572473.829858:0:261432:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572473.829860:0:261432:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xde4:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572473.829864:0:261432:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.829866:0:261432:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.829869:0:261432:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572473.829871:0:261432:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572473.829872:0:261432:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.829873:0:261432:0:(cl_object.c:917:cl_env_put()) 1@ffff9c37070767e0 00000001:00000001:0.0:1788572473.829875:0:261432:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.829883:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00000008:0.0:1788572473.829885:0:261432:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-080:[0x200002b17:0xde4:0x0](ffff9c372f2d3a88), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.829901:0:261432:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.829903:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bda0 00000080:00002000:0.0:1788572473.829905:0:261432:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-080 (ffff9c37022e8288, parent ffff9c3719d93af8, inode ffff9c372f2d3a88) hashed, 00000080:00000001:0.0:1788572473.829908:0:261432:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.829939:0:261432:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.829942:0:261432:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.829944:0:261432:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.829945:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbc0 00000080:00200000:0.0:1788572473.829947:0:261432:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.829953:0:261432:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7e00. 00800000:00000002:0.0:1788572473.829960:0:261432:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.829963:0:261432:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.829968:0:261432:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.829970:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae0 00800000:00000002:0.0:1788572473.829972:0:261432:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.829975:0:261432:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.829978:0:261432:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.829981:0:261432:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.829983:0:261432:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.829985:0:261432:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.829987:0:261432:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.829989:0:261432:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.829992:0:261432:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.829995:0:261432:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.830000:0:261432:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.830002:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b9b0 00000002:00010000:0.0:1788572473.830004:0:261432:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.830007:0:261432:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.830009:0:261432:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.830012:0:261432:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.830018:0:261432:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.830021:0:261432:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.830026:0:261432:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.830029:0:261432:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.830033:0:261432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572473.830042:0:261432:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.830046:0:261432:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.830050:0:261432:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572473.830057:0:261432:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.830059:0:261432:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.830061:0:261432:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.830063:0:261432:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.830065:0:261432:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.830067:0:261432:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.830069:0:261432:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.830073:0:261432:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.830076:0:261432:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7e00. 00000080:00000001:0.0:1788572473.830081:0:261432:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.830083:0:261432:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.830086:0:261432:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.830089:0:261432:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.830092:0:261432:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.830094:0:261432:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.830095:0:261432:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.830096:0:261432:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.830098:0:261432:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.830100:0:261432:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.830103:0:261432:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.830179:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.830180:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.830182:0:261432:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.830185:0:261432:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.830187:0:261432:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bbf8 00010000:00000001:0.0:1788572473.830190:0:261432:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.830191:0:261432:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.830192:0:261432:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.830194:0:261432:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.830196:0:261432:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.830199:0:261432:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.830202:0:261432:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.830208:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.830209:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.830211:0:261432:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.830213:0:261432:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.830215:0:261432:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.830222:0:261432:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.830225:0:261432:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.830227:0:261432:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.830230:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.830232:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.830233:0:261432:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.830237:0:261432:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.830238:0:261432:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.830240:0:261432:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.830243:0:261432:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.830245:0:261432:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.830246:0:261432:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.830247:0:261432:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.830250:0:261432:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.830252:0:261432:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.830254:0:261432:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.830258:0:261432:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.830262:0:261432:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.830264:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb88 00000080:00002000:0.0:1788572473.830266:0:261432:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 80 00000080:00000001:0.0:1788572473.830269:0:261432:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.830271:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbd0 00000080:00200000:0.0:1788572473.830272:0:261432:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572473.830278:0:261432:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.830281:0:261432:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.830283:0:261432:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.830286:0:261432:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.830288:0:261432:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.830290:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbe8 00000080:00002000:0.0:1788572473.830291:0:261432:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-080(ffff9c37022e8288) flags 0x48000e refc 0 00000080:00000001:0.0:1788572473.830294:0:261432:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.830295:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc30 00000080:00200000:0.0:1788572473.830296:0:261432:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-080, flags=9 00000080:00000001:0.0:1788572473.830307:0:261432:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572473.830311:0:261432:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572473.830313:0:261432:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.830315:0:261432:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207be40 00000080:00002000:0.0:1788572473.830316:0:261432:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-080 (ffff9c37022e8288, parent ffff9c3719d93af8, inode ffff9c372f2d3a88) hashed, 00000080:00000001:0.0:1788572473.830319:0:261432:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572473.848925:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.848932:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572473.848938:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.848944:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.848950:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c13b8 00000400:00000010:0.0:1788572473.848952:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c13b8. 00000100:00000001:0.0:1788572473.848957:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.848964:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809953152/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.848974:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.848977:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71500 x1875440809953152/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.848989:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.848990:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.852169:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.852189:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.852194:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.852196:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.852205:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.852213:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.852221:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.852226:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.852232:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.852236:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464780 00000400:00000200:0.0:1788572473.852241:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.852247:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a081 [1] + 224 00000800:00000001:0.0:1788572473.852254:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.852261:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.852264:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.852269:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.852276:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.852284:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809953152/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.852295:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec71500 x1875440809953152/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.852317:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.852324:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.852327:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572473.865224:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572473.865236:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572473.865243:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.865252:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572473.865260:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1bb0 00000400:00000010:0.0:1788572473.865263:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1bb0. 00000100:00000001:0.0:1788572473.865268:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572473.865277:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809953280/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.865289:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572473.865292:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440809953280/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.865300:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.865302:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572473.866616:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.866631:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.866736:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.866740:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.866748:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.866754:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.866762:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.866767:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.866771:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.866775:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464800 00000400:00000200:0.0:1788572473.866780:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.866785:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a089 [1] + 224 00000800:00000001:0.0:1788572473.866792:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.866799:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.866802:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.866805:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.866811:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.866817:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809953280/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.866826:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71500 x1875440809953280/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.866935:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.866944:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.866948:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.899410:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.899427:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.899432:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.899435:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.899444:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.899452:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.899461:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.899466:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.899472:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.899476:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464880 00000400:00000200:0.0:1788572473.899481:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.899488:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a091 [1] + 224 00000800:00000001:0.0:1788572473.899495:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.899502:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.899506:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.899510:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.899518:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.899528:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809953408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.899539:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6aa00 x1875440809953408/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.899560:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.899569:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.899572:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.924261:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.924279:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.924284:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.924287:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.924296:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.924305:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.924315:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.924320:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.924326:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.924330:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464900 00000400:00000200:0.0:1788572473.924335:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.924341:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a099 [1] + 224 00000800:00000001:0.0:1788572473.924350:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.924357:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.924360:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.924365:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.924373:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.924381:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809953536/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.924392:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6aa00 x1875440809953536/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.924413:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.924421:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.924424:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572473.925138:0:261436:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925142:0:261436:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.925148:0:261436:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572473.925151:0:261436:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4dc38 -> ffff9c3720b4dc38 00000020:00000001:0.0:1788572473.925153:0:261436:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4dc38 -> ffff9c3720b4dc38 00000020:00000001:0.0:1788572473.925155:0:261436:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572473.925159:0:261436:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572473.925164:0:261436:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.925165:0:261436:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.925167:0:261436:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.925168:0:261436:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.925170:0:261436:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xde6:11031 on OST[0]: s=0 (1) m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.925173:0:261436:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.925176:0:261436:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.925178:0:261436:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925180:0:261436:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925181:0:261436:0:(cl_object.c:143:cl_object_top()) ffff9c3720b4dc38 -> ffff9c3720b4dc38 00000080:00200000:0.0:1788572473.925184:0:261436:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xde6:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572473.925187:0:261436:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572473.925189:0:261436:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572473.925191:0:261436:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731327780 (ffff9c3707076128/0) at cl_lock_release():245 00000020:00000001:0.0:1788572473.925193:0:261436:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.925194:0:261436:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731327780 (ffff9c3707076128/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572473.925197:0:261436:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572473.925201:0:261436:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.925202:0:261436:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314392296 : -109714395159320 : ffff9c37207324e8) 00000020:00000001:0.0:1788572473.925204:0:261436:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.925205:0:261436:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3720e47a30 (ffff9c3707076488/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572473.925209:0:261436:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572473.925210:0:261436:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572473.925211:0:261436:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.925213:0:261436:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.925214:0:261436:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572473.925216:0:261436:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.925217:0:261436:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.925218:0:261436:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.925219:0:261436:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.925220:0:261436:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731327780 (ffff9c3707076128/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572473.925223:0:261436:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572473.925224:0:261436:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.925225:0:261436:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3720e47a30 (ffff9c3707076128/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572473.925228:0:261436:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3718db2720. 00000020:00000001:0.0:1788572473.925231:0:261436:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572473.925232:0:261436:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c3720e47a00. 00020000:00000001:0.0:1788572473.925235:0:261436:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.925236:0:261436:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.925237:0:261436:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572473.925239:0:261436:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925242:0:261436:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572473.925244:0:261436:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572473.925245:0:261436:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572473.925247:0:261436:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572473.925248:0:261436:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.925250:0:261436:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076480 00020000:00000001:0.0:1788572473.925253:0:261436:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572473.925254:0:261436:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572473.925258:0:261436:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572473.925260:0:261436:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xde6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572473.925263:0:261436:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.925265:0:261436:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.925267:0:261436:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572473.925269:0:261436:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572473.925270:0:261436:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.925271:0:261436:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076120 00000001:00000001:0.0:1788572473.925273:0:261436:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925281:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00000008:0.0:1788572473.925283:0:261436:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-082:[0x200002b17:0xde6:0x0](ffff9c372f2d21c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.925290:0:261436:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.925292:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bda0 00000080:00002000:0.0:1788572473.925293:0:261436:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-082 (ffff9c370615a360, parent ffff9c3719d93af8, inode ffff9c372f2d21c8) hashed, 00000080:00000001:0.0:1788572473.925296:0:261436:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.925324:0:261436:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.925328:0:261436:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.925330:0:261436:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.925332:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbc0 00000080:00200000:0.0:1788572473.925334:0:261436:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.925339:0:261436:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.925346:0:261436:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.925349:0:261436:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.925354:0:261436:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.925356:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae0 00800000:00000002:0.0:1788572473.925358:0:261436:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.925361:0:261436:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.925364:0:261436:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.925366:0:261436:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.925368:0:261436:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.925371:0:261436:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.925373:0:261436:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.925375:0:261436:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.925377:0:261436:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.925381:0:261436:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.925386:0:261436:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.925388:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b9b0 00000002:00010000:0.0:1788572473.925390:0:261436:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.925393:0:261436:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.925395:0:261436:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.925398:0:261436:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.925404:0:261436:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.925407:0:261436:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.925411:0:261436:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.925414:0:261436:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.925419:0:261436:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572473.925427:0:261436:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.925431:0:261436:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.925434:0:261436:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572473.925440:0:261436:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.925442:0:261436:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.925444:0:261436:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.925446:0:261436:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.925448:0:261436:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.925451:0:261436:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.925453:0:261436:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.925456:0:261436:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.925459:0:261436:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572473.925465:0:261436:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.925467:0:261436:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.925469:0:261436:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.925473:0:261436:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.925476:0:261436:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.925478:0:261436:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.925479:0:261436:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.925480:0:261436:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.925483:0:261436:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.925485:0:261436:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.925487:0:261436:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.925489:0:261436:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.925490:0:261436:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.925491:0:261436:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.925493:0:261436:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.925495:0:261436:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bbf8 00010000:00000001:0.0:1788572473.925497:0:261436:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.925498:0:261436:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.925499:0:261436:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.925500:0:261436:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.925502:0:261436:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.925504:0:261436:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.925507:0:261436:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.925513:0:261436:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.925514:0:261436:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.925515:0:261436:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.925516:0:261436:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.925519:0:261436:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.925525:0:261436:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.925526:0:261436:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.925527:0:261436:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.925530:0:261436:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.925531:0:261436:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.925532:0:261436:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.925534:0:261436:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.925535:0:261436:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.925538:0:261436:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.925540:0:261436:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.925543:0:261436:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.925544:0:261436:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.925545:0:261436:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.925547:0:261436:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.925548:0:261436:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.925550:0:261436:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.925554:0:261436:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.925557:0:261436:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.925559:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb88 00000080:00002000:0.0:1788572473.925561:0:261436:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 82 00000080:00000001:0.0:1788572473.925563:0:261436:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925565:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbd0 00000080:00200000:0.0:1788572473.925567:0:261436:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572473.925572:0:261436:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.925576:0:261436:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.925578:0:261436:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.925580:0:261436:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.925583:0:261436:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.925585:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbe8 00000080:00002000:0.0:1788572473.925586:0:261436:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-082(ffff9c370615a360) flags 0x48000e refc 0 00000080:00000001:0.0:1788572473.925589:0:261436:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.925590:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc30 00000080:00200000:0.0:1788572473.925592:0:261436:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-082, flags=9 00000080:00000001:0.0:1788572473.925601:0:261436:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572473.925604:0:261436:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572473.925607:0:261436:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.925608:0:261436:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207be40 00000080:00002000:0.0:1788572473.925609:0:261436:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-082 (ffff9c370615a360, parent ffff9c3719d93af8, inode ffff9c372f2d21c8) hashed, 00000080:00000001:0.0:1788572473.925612:0:261436:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.945744:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.945751:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.945754:0:261438:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.945758:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbb20 00000080:00200000:0.0:1788572473.945763:0:261438:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.945770:0:261438:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.945777:0:261438:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.945782:0:261438:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.945787:0:261438:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.945789:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fba40 00800000:00000002:0.0:1788572473.945792:0:261438:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.945796:0:261438:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.945799:0:261438:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.945801:0:261438:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.945804:0:261438:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.945806:0:261438:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.945808:0:261438:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.945811:0:261438:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.945813:0:261438:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.945817:0:261438:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.945822:0:261438:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.945824:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb910 00000002:00010000:0.0:1788572473.945826:0:261438:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.945830:0:261438:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.945832:0:261438:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.945835:0:261438:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.945842:0:261438:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.945846:0:261438:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.945851:0:261438:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.945855:0:261438:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.945861:0:261438:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572473.945873:0:261438:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.945878:0:261438:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.945881:0:261438:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572473.945888:0:261438:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.945890:0:261438:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.945906:0:261438:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.945908:0:261438:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.945911:0:261438:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.945913:0:261438:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.945916:0:261438:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.945919:0:261438:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.945923:0:261438:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572473.945928:0:261438:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.945930:0:261438:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.945933:0:261438:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.945936:0:261438:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.945940:0:261438:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.945942:0:261438:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.945944:0:261438:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.945946:0:261438:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.945949:0:261438:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.945952:0:261438:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.945955:0:261438:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.945957:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.945959:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.945960:0:261438:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.945962:0:261438:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.945965:0:261438:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20fbb58 00010000:00000001:0.0:1788572473.945967:0:261438:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.945969:0:261438:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.945970:0:261438:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.945972:0:261438:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.945974:0:261438:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.945976:0:261438:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.945980:0:261438:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.945986:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.945988:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.945989:0:261438:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.945991:0:261438:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.945994:0:261438:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.946002:0:261438:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.946004:0:261438:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.946007:0:261438:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.946011:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.946013:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.946014:0:261438:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946017:0:261438:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.946019:0:261438:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946022:0:261438:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946025:0:261438:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946027:0:261438:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.946028:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.946029:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.946032:0:261438:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.946034:0:261438:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.946036:0:261438:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.946040:0:261438:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.946044:0:261438:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.946046:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbae8 00000080:00002000:0.0:1788572473.946048:0:261438:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 82 00000080:00000001:0.0:1788572473.946050:0:261438:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.946052:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbb30 00000080:00200000:0.0:1788572473.946054:0:261438:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572473.946059:0:261438:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.946063:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.946064:0:261438:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.946067:0:261438:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572473.946074:0:261438:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f960. 00000020:00000001:0.0:1788572473.946078:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbae8 00000080:00200000:0.0:1788572473.946080:0:261438:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-083, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572473.946083:0:261438:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572473.946084:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb998 00000080:00200000:0.0:1788572473.946087:0:261438:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-083, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572473.946092:0:261438:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572473.946095:0:261438:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572473.946098:0:261438:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.946100:0:261438:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.946101:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb8b8 00800000:00000002:0.0:1788572473.946103:0:261438:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-083' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.946106:0:261438:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.946108:0:261438:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.946110:0:261438:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.946111:0:261438:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.946113:0:261438:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.946114:0:261438:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.946116:0:261438:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572473.946118:0:261438:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.946121:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb850 00800000:00000002:0.0:1788572473.946122:0:261438:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-083' -> mds #0 00000002:00000001:0.0:1788572473.946125:0:261438:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.946127:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb788 00000002:00010000:0.0:1788572473.946129:0:261438:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-083,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572473.946133:0:261438:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572473.946136:0:261438:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572473.946140:0:261438:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572473.946144:0:261438:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=43 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572473.946150:0:261438:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.946154:0:261438:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.946157:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.946158:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.946159:0:261438:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.946162:0:261438:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.946164:0:261438:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.946167:0:261438:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.946173:0:261438:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572473.946180:0:261438:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946185:0:261438:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.946188:0:261438:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946189:0:261438:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.946191:0:261438:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.946193:0:261438:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572473.946199:0:261438:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531979648 : -109714177571968 : ffff9c372d6b4380) 00010000:00000001:0.0:1788572473.946203:0:261438:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.946206:0:261438:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572473.946209:0:261438:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572473.946212:0:261438:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.946214:0:261438:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.946217:0:261438:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788572473.946221:0:261438:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.946224:0:261438:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168771 to hash 00000020:00000001:0.0:1788572473.946226:0:261438:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.946227:0:261438:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572473.946231:0:261438:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572473.946233:0:261438:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.946235:0:261438:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.946238:0:261438:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168771 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261438 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.946246:0:261438:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168771 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261438 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.946255:0:261438:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168771 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261438 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572473.946261:0:261438:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572473.946263:0:261438:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572473.946269:0:261438:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5cc0. 00000100:00000001:0.0:1788572473.946271:0:261438:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716480 : -109714420835136 : ffff9c371eeb5cc0) 00000100:00000001:0.0:1788572473.946274:0:261438:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.946276:0:261438:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531979648 : -109714177571968 : ffff9c372d6b4380) 00000100:00000001:0.0:1788572473.946280:0:261438:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.946284:0:261438:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.946286:0:261438:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572473.946288:0:261438:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.946299:0:261438:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4380 x1875440809953664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.946308:0:261438:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.946310:0:261438:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.946313:0:261438:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.946316:0:261438:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.946318:0:261438:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.946322:0:261438:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4380 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261438:1875440809953664:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.946326:0:261438:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.946331:0:261438:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.946333:0:261438:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.946335:0:261438:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.946340:0:261438:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef800. 02000000:00000001:0.0:1788572473.946342:0:261438:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.946350:0:261438:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.946354:0:261438:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8528. 00000400:00000010:0.0:1788572473.946361:0:261438:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572473.946367:0:261438:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809953664, portal 10 00000100:00000001:0.0:1788572473.946369:0:261438:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.946370:0:261438:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531979648 : -109714177571968 : ffff9c372d6b4380) 00000100:00000040:0.0:1788572473.946376:0:261438:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4380 x1875440809953664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.946384:0:261438:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.946387:0:261438:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.946390:0:261438:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31220. 00000100:00000200:0.0:1788572473.946392:0:261438:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809953664, offset 0 00000400:00000200:0.0:1788572473.946398:0:261438:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.946406:0:261438:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.946413:0:261438:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.946418:0:261438:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.946422:0:261438:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.946425:0:261438:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.946428:0:261438:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.946431:0:261438:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539830:539830:256:4294967295] 192.168.202.148@tcp LPNI seq info [539795:539795:8:4294967295] 00000400:00000200:0.0:1788572473.946435:0:261438:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.946445:0:261438:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.946452:0:261438:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.946458:0:261438:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572473.946464:0:261438:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.946472:0:261438:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.946476:0:261438:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.946496:0:261438:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.946500:0:261438:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15022 usecs (16 jiffies) 00000400:00000200:0.0:1788572473.946503:0:261438:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.946506:0:261438:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307612968 00000100:00000001:0.0:1788572473.946508:0:261438:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.946511:0:261438:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307612968 00000100:00000001:0.0:1788572473.946512:0:261438:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.946515:0:261438:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.946517:0:261438:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.946519:0:261438:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.946521:0:261438:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572473.946524:0:261438:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5cc0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572473.946527:0:261438:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.946530:0:261438:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.965275:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.965294:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.965299:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.965303:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.965313:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.965320:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.965330:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.965336:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.965341:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.965346:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464980 00000400:00000200:0.0:1788572473.965351:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.965358:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a0a1 [1] + 224 00000800:00000001:0.0:1788572473.965366:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.965388:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.965392:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.965397:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.965405:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.965415:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4380 x1875440809953664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.965427:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b4380 x1875440809953664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572558 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.965455:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.965464:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.965468:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788572473.974120:0:261438:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572473.974132:0:261438:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=6 obd=lustre-OST0001-osc-ffff9c3711155800 00010000:00000001:0.0:1788572473.974138:0:261438:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572473.974144:0:261438:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.974150:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.974153:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.974155:0:261438:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.974159:0:261438:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572473.974161:0:261438:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572473.974166:0:261438:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572473.974175:0:261438:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6400. 00000100:00000001:0.0:1788572473.974182:0:261438:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.974187:0:261438:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.974189:0:261438:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.974191:0:261438:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572473.974193:0:261438:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.974194:0:261438:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.974198:0:261438:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00010000:00000001:0.0:1788572473.974202:0:261438:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572473.974205:0:261438:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572473.974213:0:261438:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3710a87500. 00010000:00000010:0.0:1788572473.974216:0:261438:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3710a87a00. 00010000:00000001:0.0:1788572473.974220:0:261438:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572473.974223:0:261438:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788572473.974228:0:261438:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572473.974231:0:261438:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168778 to hash 00000020:00000001:0.0:1788572473.974233:0:261438:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572473.974234:0:261438:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000010:0.0:1788572473.974238:0:261438:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091680. 00010000:00000001:0.0:1788572473.974241:0:261438:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788572473.974244:0:261438:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.974247:0:261438:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.974251:0:261438:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cb6ad56d/0x1441586093168778 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5212:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261438 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572473.974264:0:261438:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000cb6ad56d/0x1441586093168778 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x5212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261438 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572473.974274:0:261438:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.974279:0:261438:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572473.974283:0:261438:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.974296:0:261438:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4a80 x1875440809953792/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572473.974331:0:261438:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572473.974336:0:261438:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.974339:0:261438:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.974342:0:261438:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.974344:0:261438:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.974347:0:261438:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572473.974349:0:261438:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572473.974351:0:261438:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572473.974352:0:261438:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572473.974361:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.974365:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.974367:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572473.974373:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440809953792/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.974382:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572473.974384:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.974388:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572473.974391:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572473.974393:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572473.974397:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809953792:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572473.974402:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572473.974408:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572473.974410:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572473.974412:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572473.974417:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788572473.974419:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572473.974428:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572473.974432:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8dc0. 00000400:00000010:0.0:1788572473.974440:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572473.974447:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809953792, portal 4 00000100:00000001:0.0:1788572473.974450:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572473.974452:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000040:0.0:1788572473.974457:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4a80 x1875440809953792/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.974464:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572473.974467:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572473.974470:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788572473.974473:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809953792, offset 0 00000400:00000200:0.0:1788572473.974480:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572473.974489:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.974496:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572473.974502:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572473.974505:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572473.974508:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572473.974511:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572473.974515:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539831:539831:256:4294967295] 192.168.202.148@tcp LPNI seq info [539796:539796:8:4294967295] 00000400:00000200:0.0:1788572473.974519:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.974529:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572473.974535:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572473.974540:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572473.974546:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572473.974553:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572473.974557:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572473.974566:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572473.974569:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 175976 usecs (176 jiffies) 00000400:00000200:0.0:1788572473.974573:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572473.974576:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307613150 00000100:00000001:0.0:1788572473.974579:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.974583:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307613150 00000100:00000001:0.0:1788572473.974584:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572473.974586:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572473.974589:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.974592:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.974594:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572473.974600:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572473.974602:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572473.974604:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572473.974606:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572473.974608:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572473.974610:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572473.983215:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.983232:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572473.983236:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.983239:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.983248:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.983255:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572473.983264:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.983269:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.983274:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.983278:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464a00 00000400:00000200:0.0:1788572473.983283:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572473.983289:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a0b1 [1] + 224 00000800:00000001:0.0:1788572473.983296:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.983301:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572473.983304:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572473.983308:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572473.983314:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572473.983321:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4a80 x1875440809953792/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572473.983330:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c372d6b4a80 x1875440809953792/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572489 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572473.983356:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572473.983363:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572473.983366:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572473.988695:0:261438:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.988706:0:261438:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.988714:0:261438:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572473.988720:0:261438:0:(cl_object.c:143:cl_object_top()) ffff9c3731068530 -> ffff9c3731068530 00000020:00000001:0.0:1788572473.988723:0:261438:0:(cl_object.c:143:cl_object_top()) ffff9c3731068530 -> ffff9c3731068530 00000020:00000001:0.0:1788572473.988726:0:261438:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572473.988732:0:261438:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572473.988736:0:261438:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.988738:0:261438:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572473.988740:0:261438:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572473.988741:0:261438:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.988742:0:261438:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xde7:11031 on OST[1]: s=0 (1) m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.988746:0:261438:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572473.988748:0:261438:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572473.988750:0:261438:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.988752:0:261438:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.988753:0:261438:0:(cl_object.c:143:cl_object_top()) ffff9c3731068530 -> ffff9c3731068530 00000080:00200000:0.0:1788572473.988755:0:261438:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xde7:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572473.988757:0:261438:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572473.988760:0:261438:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572473.988761:0:261438:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731326750 (ffff9c3707076638/0) at cl_lock_release():245 00000020:00000001:0.0:1788572473.988764:0:261438:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.988765:0:261438:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731326750 (ffff9c3707076638/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572473.988769:0:261438:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572473.988772:0:261438:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572473.988774:0:261438:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314388208 : -109714395163408 : ffff9c37207314f0) 00000020:00000001:0.0:1788572473.988776:0:261438:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572473.988777:0:261438:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c373192a7b0 (ffff9c3707076c68/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572473.988781:0:261438:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572473.988782:0:261438:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572473.988784:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.988786:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572473.988788:0:261438:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572473.988790:0:261438:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.988791:0:261438:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.988792:0:261438:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572473.988793:0:261438:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.988794:0:261438:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731326750 (ffff9c3707076638/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572473.988796:0:261438:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572473.988797:0:261438:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572473.988798:0:261438:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c373192a7b0 (ffff9c3707076638/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572473.988801:0:261438:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c370313a720. 00000020:00000001:0.0:1788572473.988807:0:261438:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572473.988808:0:261438:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c373192a780. 00020000:00000001:0.0:1788572473.988812:0:261438:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.988813:0:261438:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572473.988814:0:261438:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572473.988816:0:261438:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.988819:0:261438:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572473.988822:0:261438:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572473.988824:0:261438:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572473.988825:0:261438:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572473.988826:0:261438:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.988829:0:261438:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076c60 00020000:00000001:0.0:1788572473.988832:0:261438:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572473.988834:0:261438:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572473.988837:0:261438:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572473.988839:0:261438:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xde7:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572473.988843:0:261438:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572473.988845:0:261438:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.988848:0:261438:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572473.988849:0:261438:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572473.988850:0:261438:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572473.988852:0:261438:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076630 00000001:00000001:0.0:1788572473.988854:0:261438:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.988864:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbd08 00000080:00000008:0.0:1788572473.988867:0:261438:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-083:[0x200002b17:0xde7:0x0](ffff9c37087be3c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572473.988873:0:261438:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.988875:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbda0 00000080:00002000:0.0:1788572473.988876:0:261438:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-083 (ffff9c370804f510, parent ffff9c3719d93af8, inode ffff9c37087be3c8) hashed, 00000080:00000001:0.0:1788572473.988879:0:261438:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.988926:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.988930:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572473.988932:0:261438:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572473.988934:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbc0 00000080:00200000:0.0:1788572473.988936:0:261438:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572473.988943:0:261438:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6000. 00800000:00000002:0.0:1788572473.988951:0:261438:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572473.988955:0:261438:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572473.988960:0:261438:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.988961:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbae0 00800000:00000002:0.0:1788572473.988963:0:261438:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.988967:0:261438:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572473.988971:0:261438:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572473.988974:0:261438:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572473.988976:0:261438:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572473.988978:0:261438:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572473.988980:0:261438:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.988982:0:261438:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572473.988985:0:261438:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572473.988988:0:261438:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572473.988993:0:261438:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572473.988995:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fb9b0 00000002:00010000:0.0:1788572473.988997:0:261438:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572473.989000:0:261438:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572473.989002:0:261438:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572473.989005:0:261438:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572473.989012:0:261438:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572473.989015:0:261438:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572473.989020:0:261438:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572473.989025:0:261438:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572473.989030:0:261438:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572473.989040:0:261438:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572473.989046:0:261438:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572473.989049:0:261438:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572473.989056:0:261438:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572473.989057:0:261438:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572473.989059:0:261438:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.989061:0:261438:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572473.989063:0:261438:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.989065:0:261438:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572473.989067:0:261438:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572473.989070:0:261438:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572473.989073:0:261438:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6000. 00000080:00000001:0.0:1788572473.989081:0:261438:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572473.989082:0:261438:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.989613:0:261438:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572473.989619:0:261438:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572473.989623:0:261438:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572473.989625:0:261438:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572473.989628:0:261438:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.989630:0:261438:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.989633:0:261438:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.989661:0:261438:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.989664:0:261438:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.989667:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.989669:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572473.989670:0:261438:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572473.989672:0:261438:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572473.989675:0:261438:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc20fbbf8 00010000:00000001:0.0:1788572473.989677:0:261438:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572473.989678:0:261438:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572473.989679:0:261438:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572473.989681:0:261438:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.989683:0:261438:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572473.989686:0:261438:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572473.989689:0:261438:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.989697:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.989698:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.989699:0:261438:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572473.989702:0:261438:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572473.989704:0:261438:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572473.989711:0:261438:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572473.989715:0:261438:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572473.989717:0:261438:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572473.989722:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.989723:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572473.989725:0:261438:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.989728:0:261438:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572473.989730:0:261438:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.989732:0:261438:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.989734:0:261438:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572473.989736:0:261438:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572473.989738:0:261438:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572473.989738:0:261438:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572473.989742:0:261438:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572473.989743:0:261438:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572473.989745:0:261438:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572473.989750:0:261438:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.989754:0:261438:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.989757:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbb88 00000080:00002000:0.0:1788572473.989758:0:261438:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 83 00000080:00000001:0.0:1788572473.989761:0:261438:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.989763:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbd0 00000080:00200000:0.0:1788572473.989764:0:261438:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572473.989770:0:261438:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572473.989774:0:261438:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572473.989775:0:261438:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572473.989778:0:261438:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572473.989780:0:261438:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572473.989782:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbbe8 00000080:00002000:0.0:1788572473.989783:0:261438:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-083(ffff9c370804f510) flags 0x48000e refc 0 00000080:00000001:0.0:1788572473.989785:0:261438:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572473.989787:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbc30 00000080:00200000:0.0:1788572473.989788:0:261438:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-083, flags=9 00000080:00000001:0.0:1788572473.989799:0:261438:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572473.989802:0:261438:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572473.989805:0:261438:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572473.989806:0:261438:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20fbe40 00000080:00002000:0.0:1788572473.989808:0:261438:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-083 (ffff9c370804f510, parent ffff9c3719d93af8, inode ffff9c37087be3c8) hashed, 00000080:00000001:0.0:1788572473.989811:0:261438:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572474.053149:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.053168:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.053172:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.053175:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.053184:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.053191:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.053201:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.053205:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.053210:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.053214:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464a80 00000400:00000200:0.0:1788572474.053220:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.053227:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a0c1 [1] + 224 00000800:00000001:0.0:1788572474.053233:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.053465:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.053468:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.053474:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.053482:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.053491:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809953920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.053502:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6aa00 x1875440809953920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.053518:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.053526:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.053530:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.070160:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.070180:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.070184:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.070187:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.070195:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.070203:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.070212:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.070217:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.070222:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.070226:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464b00 00000400:00000200:0.0:1788572474.070231:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.070237:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a0c9 [1] + 224 00000800:00000001:0.0:1788572474.070243:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.070250:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.070253:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.070257:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.070264:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.070272:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809954048/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.070282:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71f80 x1875440809954048/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.070303:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.070310:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.070314:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.091036:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.091047:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572474.091053:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.091059:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.091066:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572474.091068:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572474.091072:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.091078:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809954176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.091088:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.091092:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6300 x1875440809954176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.091098:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.091100:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.096966:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.096984:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.096988:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.096990:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.096999:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.097007:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.097016:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.097020:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.097027:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.097030:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464b80 00000400:00000200:0.0:1788572474.097034:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.097040:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a0d1 [1] + 224 00000800:00000001:0.0:1788572474.097048:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.097053:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.097057:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.097061:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.097068:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.097077:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809954176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.097086:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6300 x1875440809954176/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.097107:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.097115:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.097118:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572474.099188:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572474.099191:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572474.099194:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572474.099200:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.099209:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572474.099211:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.099215:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572474.099217:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572474.099220:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572474.099224:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809954304:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572474.099228:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572474.099233:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572474.099235:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.099238:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572474.099243:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788572474.099245:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572474.099252:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572474.099255:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c85d8. 00000400:00000010:0.0:1788572474.099261:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572474.099267:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809954304, portal 4 00000100:00000001:0.0:1788572474.099269:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572474.099271:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291929344 : -109714417622272 : ffff9c371f1c6300) 00000100:00000040:0.0:1788572474.099276:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.099283:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572474.099285:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572474.099288:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788572474.099290:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809954304, offset 0 00000400:00000200:0.0:1788572474.099297:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572474.099304:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.099311:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572474.099316:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572474.099321:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572474.099324:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572474.099327:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572474.099331:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539835:539835:256:4294967295] 192.168.202.148@tcp LPNI seq info [539800:539800:8:4294967295] 00000400:00000200:0.0:1788572474.099336:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.099345:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572474.099351:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572474.099357:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572474.099364:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572474.099371:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572474.099376:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572474.099386:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572474.099390:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 175797 usecs (176 jiffies) 00000400:00000200:0.0:1788572474.099393:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572474.099396:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307613281 00000100:00000001:0.0:1788572474.099399:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.099401:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307613281 00000100:00000001:0.0:1788572474.099403:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.099406:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572474.099408:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.099411:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.099413:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572474.099504:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572474.099506:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572474.099509:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572474.099510:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572474.099512:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.099514:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572474.099874:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.099878:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788572474.099883:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.099889:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.099905:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed317f8 00000400:00000010:0.0:1788572474.099908:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed317f8. 00000100:00000001:0.0:1788572474.099911:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.099916:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.099926:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.099929:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.099936:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.099938:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.107889:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.107918:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.107923:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.107925:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.107935:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.107942:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.107951:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.107956:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.107961:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.107966:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464c00 00000400:00000200:0.0:1788572474.107971:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.107977:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a0e1 [1] + 224 00000800:00000001:0.0:1788572474.107985:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.107992:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.107995:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.107999:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.108008:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.108018:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.108030:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c6300 x1875440809954304/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.108057:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.108065:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.108069:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.133970:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.133974:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572474.133979:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.133984:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.133989:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572474.133992:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572474.133996:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.134000:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809954432/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.134009:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.134012:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6300 x1875440809954432/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.134018:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.134020:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.144241:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.144261:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.144265:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.144269:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.144278:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.144286:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.144296:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.144302:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.144307:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.144312:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464c80 00000400:00000200:0.0:1788572474.144317:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.144322:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a0f1 [1] + 224 00000800:00000001:0.0:1788572474.144329:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.144336:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.144339:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.144343:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.144351:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.144359:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809954432/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.144369:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c6300 x1875440809954432/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.144445:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.144455:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.144458:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.153909:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.153913:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572474.153918:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.153925:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.153930:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1110 00000400:00000010:0.0:1788572474.153933:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1110. 00000100:00000001:0.0:1788572474.153937:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.153941:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809954560/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.153949:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.153952:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440809954560/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.153957:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.153959:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.156876:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.156904:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.156909:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.156912:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.156921:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.156928:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.156937:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.156941:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.156947:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.156951:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464d00 00000400:00000200:0.0:1788572474.156956:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.156961:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a101 [1] + 224 00000800:00000001:0.0:1788572474.156968:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.156975:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.156977:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.156982:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.156990:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.156999:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809954560/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.157009:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec72680 x1875440809954560/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.157035:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.157043:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.157046:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.193298:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.193316:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.193320:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.193322:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.193332:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.193340:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.193349:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.193354:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.193359:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.193363:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464d80 00000400:00000200:0.0:1788572474.193368:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.193373:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a109 [1] + 224 00000800:00000001:0.0:1788572474.193380:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.193388:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.193392:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.193395:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.193402:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.193411:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809954688/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.193423:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ec72680 x1875440809954688/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.193451:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.193459:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.193462:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.212237:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.212257:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.212261:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.212264:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.212273:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.212280:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.212291:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.212295:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.212300:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.212305:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464e00 00000400:00000200:0.0:1788572474.212310:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.212315:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a111 [1] + 224 00000800:00000001:0.0:1788572474.212322:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.212333:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.212336:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.212340:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.212347:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.212356:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809954816/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.212366:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70e00 x1875440809954816/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.212390:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.212398:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.212401:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572474.212965:0:261446:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.212976:0:261446:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.212989:0:261446:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572474.212995:0:261446:0:(cl_object.c:143:cl_object_top()) ffff9c37043489e0 -> ffff9c37043489e0 00000020:00000001:0.0:1788572474.213004:0:261446:0:(cl_object.c:143:cl_object_top()) ffff9c37043489e0 -> ffff9c37043489e0 00000020:00000001:0.0:1788572474.213011:0:261446:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572474.213019:0:261446:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572474.213030:0:261446:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572474.213035:0:261446:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.213041:0:261446:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572474.213045:0:261446:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572474.213051:0:261446:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdeb:11031 on OST[1]: s=0 (1) m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572474.213069:0:261446:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572474.213076:0:261446:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572468 a=1788572468 c=1788572468 b=0 00020000:00000001:0.0:1788572474.213081:0:261446:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213084:0:261446:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213100:0:261446:0:(cl_object.c:143:cl_object_top()) ffff9c37043489e0 -> ffff9c37043489e0 00000080:00200000:0.0:1788572474.213114:0:261446:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdeb:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572474.213120:0:261446:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572474.213127:0:261446:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572474.213133:0:261446:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572474.213143:0:261446:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572474.213147:0:261446:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572474.213157:0:261446:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572474.213166:0:261446:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572474.213171:0:261446:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314383536 : -109714395168080 : ffff9c37207302b0) 00000020:00000001:0.0:1788572474.213183:0:261446:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572474.213185:0:261446:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c37320cff30 (ffff9c3718da6ea8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572474.213195:0:261446:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572474.213197:0:261446:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572474.213201:0:261446:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.213205:0:261446:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572474.213209:0:261446:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572474.213214:0:261446:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.213218:0:261446:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.213222:0:261446:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.213224:0:261446:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572474.213226:0:261446:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572474.213236:0:261446:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572474.213241:0:261446:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572474.213248:0:261446:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c37320cff30 (ffff9c3707076cf8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572474.213263:0:261446:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c3703eb8720. 00000020:00000001:0.0:1788572474.213272:0:261446:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572474.213277:0:261446:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c37320cff00. 00020000:00000001:0.0:1788572474.213285:0:261446:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572474.213288:0:261446:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572474.213291:0:261446:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572474.213296:0:261446:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213305:0:261446:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572474.213315:0:261446:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572474.213319:0:261446:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572474.213322:0:261446:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572474.213325:0:261446:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572474.213331:0:261446:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6ea0 00020000:00000001:0.0:1788572474.213339:0:261446:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572474.213344:0:261446:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572474.213350:0:261446:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572474.213355:0:261446:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdeb:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572474.213362:0:261446:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572474.213365:0:261446:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.213371:0:261446:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572474.213378:0:261446:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572474.213383:0:261446:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572474.213386:0:261446:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076cf0 00000001:00000001:0.0:1788572474.213391:0:261446:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213400:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bd08 00000080:00000008:0.0:1788572474.213402:0:261446:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-087:[0x200002b17:0xdeb:0x0](ffff9c372f2d00c8), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572474.213409:0:261446:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572474.213411:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bda0 00000080:00002000:0.0:1788572474.213413:0:261446:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-087 (ffff9c37022e85e8, parent ffff9c3719d93af8, inode ffff9c372f2d00c8) hashed, 00000080:00000001:0.0:1788572474.213416:0:261446:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.213481:0:261446:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.213485:0:261446:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.213487:0:261446:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572474.213488:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbc0 00000080:00200000:0.0:1788572474.213490:0:261446:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572474.213498:0:261446:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572474.213504:0:261446:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572474.213509:0:261446:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572474.213512:0:261446:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.213515:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bae0 00800000:00000002:0.0:1788572474.213517:0:261446:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.213521:0:261446:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572474.213524:0:261446:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.213526:0:261446:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.213529:0:261446:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.213531:0:261446:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.213534:0:261446:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.213536:0:261446:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.213539:0:261446:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.213541:0:261446:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572474.213547:0:261446:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.213549:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207b9b0 00000002:00010000:0.0:1788572474.213551:0:261446:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572474.213555:0:261446:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572474.213556:0:261446:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572474.213559:0:261446:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572474.213566:0:261446:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572474.213569:0:261446:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.213573:0:261446:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.213578:0:261446:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572474.213582:0:261446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572474.213593:0:261446:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572474.213599:0:261446:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.213603:0:261446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572474.213610:0:261446:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572474.213613:0:261446:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572474.213615:0:261446:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.213617:0:261446:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.213619:0:261446:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.213621:0:261446:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572474.213624:0:261446:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572474.213627:0:261446:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572474.213630:0:261446:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6400. 00000080:00000001:0.0:1788572474.213664:0:261446:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572474.213666:0:261446:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.213669:0:261446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572474.213672:0:261446:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572474.213675:0:261446:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572474.213677:0:261446:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572474.213679:0:261446:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.213681:0:261446:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.213684:0:261446:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.213686:0:261446:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.213689:0:261446:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.213692:0:261446:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.213693:0:261446:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572474.213694:0:261446:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572474.213696:0:261446:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.213698:0:261446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc207bbf8 00010000:00000001:0.0:1788572474.213700:0:261446:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.213701:0:261446:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.213702:0:261446:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.213703:0:261446:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.213705:0:261446:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.213707:0:261446:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572474.213710:0:261446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.213715:0:261446:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.213716:0:261446:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.213718:0:261446:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572474.213720:0:261446:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572474.213722:0:261446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.213729:0:261446:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572474.213732:0:261446:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572474.213735:0:261446:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.213739:0:261446:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.213740:0:261446:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.213741:0:261446:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.213744:0:261446:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572474.213746:0:261446:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.213748:0:261446:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.213750:0:261446:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.213752:0:261446:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572474.213753:0:261446:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.213754:0:261446:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572474.213757:0:261446:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572474.213759:0:261446:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.213761:0:261446:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572474.213764:0:261446:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.213768:0:261446:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.213770:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bb88 00000080:00002000:0.0:1788572474.213771:0:261446:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 87 00000080:00000001:0.0:1788572474.213774:0:261446:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213776:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbd0 00000080:00200000:0.0:1788572474.213778:0:261446:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572474.213783:0:261446:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572474.213787:0:261446:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572474.213789:0:261446:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572474.213793:0:261446:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.213796:0:261446:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.213799:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bbe8 00000080:00002000:0.0:1788572474.213802:0:261446:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-087(ffff9c37022e85e8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572474.213804:0:261446:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.213807:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207bc30 00000080:00200000:0.0:1788572474.213809:0:261446:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-087, flags=9 00000080:00000001:0.0:1788572474.213823:0:261446:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572474.213827:0:261446:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572474.213829:0:261446:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572474.213831:0:261446:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc207be40 00000080:00002000:0.0:1788572474.213833:0:261446:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-087 (ffff9c37022e85e8, parent ffff9c3719d93af8, inode ffff9c372f2d00c8) hashed, 00000080:00000001:0.0:1788572474.213835:0:261446:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572474.251194:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.251204:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572474.251213:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.251222:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.251231:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3dd0 00000400:00000010:0.0:1788572474.251235:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3dd0. 00000100:00000001:0.0:1788572474.251240:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.251247:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7480 x1875440809954944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.251262:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.251266:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7480 x1875440809954944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.251273:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.251275:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.278709:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.278738:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.278744:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.278746:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.278758:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.278769:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.278780:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.278785:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.278792:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.278797:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464e80 00000400:00000200:0.0:1788572474.278803:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.278811:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a119 [1] + 224 00000800:00000001:0.0:1788572474.278821:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.278829:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.278834:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.278839:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.278852:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.278868:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7480 x1875440809954944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.278887:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c7480 x1875440809954944/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.278924:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.278935:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.278939:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.288783:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.288790:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572474.288796:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.288803:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.288810:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1660 00000400:00000010:0.0:1788572474.288813:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1660. 00000100:00000001:0.0:1788572474.288817:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.288822:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71180 x1875440809955072/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.288834:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.288838:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71180 x1875440809955072/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.288845:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.288847:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.323004:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.323035:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.323040:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.323042:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.323055:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.323065:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.323079:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.323087:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.323093:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.323099:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417464f00 00000400:00000200:0.0:1788572474.323105:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.323113:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a129 [1] + 224 00000800:00000001:0.0:1788572474.323123:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.323139:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.323143:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.323149:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.323159:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.323171:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71180 x1875440809955072/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.323185:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71180 x1875440809955072/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.323213:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.323223:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.323227:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.358333:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.358340:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572474.358346:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.358352:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.358359:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1880 00000400:00000010:0.0:1788572474.358361:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1880. 00000100:00000001:0.0:1788572474.358366:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.358371:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809955200/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.358383:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.358387:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6aa00 x1875440809955200/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.358395:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.358396:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.376198:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.376227:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.376232:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.376235:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.376247:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.376258:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.376273:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.376280:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.376286:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.376292:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417464f80 00000400:00000200:0.0:1788572474.376298:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.376307:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a131 [1] + 224 00000800:00000001:0.0:1788572474.376316:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.376331:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.376335:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.376341:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.376350:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.376362:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809955200/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.376375:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6aa00 x1875440809955200/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.376401:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.376412:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.376415:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.389829:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.389839:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572474.389847:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.389856:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.389865:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1908 00000400:00000010:0.0:1788572474.389868:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1908. 00000100:00000001:0.0:1788572474.389873:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.389880:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73b80 x1875440809955328/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.389905:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.389910:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73b80 x1875440809955328/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.389919:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.389921:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.393038:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.393060:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.393065:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.393068:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.393077:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.393085:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.393157:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.393163:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.393168:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.393172:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465000 00000400:00000200:0.0:1788572474.393177:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.393184:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a139 [1] + 224 00000800:00000001:0.0:1788572474.393192:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.393199:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.393203:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.393207:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.393215:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.393224:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73b80 x1875440809955328/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.393235:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec73b80 x1875440809955328/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.393254:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.393262:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.393265:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572474.441018:0:261453:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.441030:0:261453:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.441033:0:261453:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572474.441039:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7b20 00000080:00200000:0.0:1788572474.441045:0:261453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572474.441055:0:261453:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572474.441063:0:261453:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572474.441068:0:261453:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572474.441075:0:261453:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.441077:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7a40 00800000:00000002:0.0:1788572474.441092:0:261453:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.441097:0:261453:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572474.441101:0:261453:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.441105:0:261453:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.441108:0:261453:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.441110:0:261453:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.441113:0:261453:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.441115:0:261453:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.441118:0:261453:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.441122:0:261453:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572474.441128:0:261453:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.441130:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7910 00000002:00010000:0.0:1788572474.441132:0:261453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572474.441136:0:261453:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572474.441139:0:261453:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572474.441143:0:261453:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572474.441152:0:261453:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572474.441155:0:261453:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.441162:0:261453:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.441167:0:261453:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572474.441172:0:261453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572474.441185:0:261453:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572474.441192:0:261453:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.441196:0:261453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572474.441203:0:261453:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572474.441206:0:261453:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572474.441209:0:261453:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.441211:0:261453:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.441213:0:261453:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.441216:0:261453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572474.441219:0:261453:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572474.441222:0:261453:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572474.441226:0:261453:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b6400. 00000080:00000001:0.0:1788572474.441237:0:261453:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572474.441239:0:261453:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.441242:0:261453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572474.441246:0:261453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572474.441250:0:261453:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572474.441252:0:261453:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572474.441255:0:261453:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.441258:0:261453:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.441262:0:261453:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.441265:0:261453:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.441268:0:261453:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.441272:0:261453:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.441274:0:261453:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572474.441275:0:261453:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572474.441277:0:261453:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.441280:0:261453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc21d7b58 00010000:00000001:0.0:1788572474.441282:0:261453:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.441284:0:261453:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.441285:0:261453:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.441287:0:261453:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.441290:0:261453:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.441292:0:261453:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572474.441295:0:261453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.441301:0:261453:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.441302:0:261453:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.441304:0:261453:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572474.441306:0:261453:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572474.441309:0:261453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.441323:0:261453:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572474.441326:0:261453:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572474.441329:0:261453:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.441334:0:261453:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.441335:0:261453:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.441337:0:261453:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441341:0:261453:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572474.441344:0:261453:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441346:0:261453:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441349:0:261453:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441351:0:261453:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572474.441352:0:261453:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.441353:0:261453:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572474.441357:0:261453:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572474.441359:0:261453:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.441362:0:261453:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572474.441367:0:261453:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.441371:0:261453:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.441374:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7ae8 00000080:00002000:0.0:1788572474.441375:0:261453:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 89 00000080:00000001:0.0:1788572474.441378:0:261453:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.441380:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7b30 00000080:00200000:0.0:1788572474.441382:0:261453:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=24 00000080:00000002:0.0:1788572474.441388:0:261453:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572474.441392:0:261453:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572474.441394:0:261453:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572474.441397:0:261453:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572474.441407:0:261453:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c370310f6c0. 00000020:00000001:0.0:1788572474.441412:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7ae8 00000080:00200000:0.0:1788572474.441415:0:261453:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f434.sanity-090, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), flags=8 00000080:00000001:0.0:1788572474.441418:0:261453:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572474.441420:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7998 00000080:00200000:0.0:1788572474.441422:0:261453:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f434.sanity-090, dir=[0x200002b17:0xd94:0x0](ffff9c37087b9148), intent=getattr 00000080:00000010:0.0:1788572474.441430:0:261453:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b6400. 00800000:00000002:0.0:1788572474.441434:0:261453:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c3704c3a040 2 00000080:00000001:0.0:1788572474.441438:0:261453:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572474.441440:0:261453:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.441441:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d78b8 00800000:00000002:0.0:1788572474.441443:0:261453:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f434.sanity-090' on [0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572474.441457:0:261453:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572474.441461:0:261453:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.441462:0:261453:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.441463:0:261453:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.441465:0:261453:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.441466:0:261453:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.441468:0:261453:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xd94:0x0] 00800000:00000001:0.0:1788572474.441469:0:261453:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.441471:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7850 00800000:00000002:0.0:1788572474.441473:0:261453:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x0:0x0:0x0], name='f434.sanity-090' -> mds #0 00000002:00000001:0.0:1788572474.441476:0:261453:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.441477:0:261453:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21d7788 00000002:00010000:0.0:1788572474.441479:0:261453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f434.sanity-090,[0x0:0x0:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572474.441483:0:261453:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572474.441486:0:261453:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572474.441491:0:261453:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572474.441496:0:261453:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572474.441503:0:261453:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572474.441507:0:261453:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.441510:0:261453:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.441511:0:261453:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.441512:0:261453:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.441515:0:261453:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572474.441518:0:261453:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572474.441521:0:261453:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572474.441529:0:261453:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572474.441537:0:261453:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441543:0:261453:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572474.441545:0:261453:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441546:0:261453:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572474.441547:0:261453:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.441550:0:261453:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572474.441558:0:261453:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00010000:00000001:0.0:1788572474.441562:0:261453:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572474.441564:0:261453:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572474.441569:0:261453:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3730c25600 count: 2 00010000:00000001:0.0:1788572474.441571:0:261453:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.441573:0:261453:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572474.441577:0:261453:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788572474.441581:0:261453:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572474.441584:0:261453:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x14415860931687d3 to hash 00000020:00000001:0.0:1788572474.441586:0:261453:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572474.441588:0:261453:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572474.441592:0:261453:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572474.441595:0:261453:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.441596:0:261453:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.441600:0:261453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x14415860931687d3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 261453 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572474.441607:0:261453:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x14415860931687d3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261453 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572474.441619:0:261453:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x14415860931687d3 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xd94:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 261453 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572474.441625:0:261453:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572474.441627:0:261453:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572474.441670:0:261453:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371eeb5cc0. 00000100:00000001:0.0:1788572474.441673:0:261453:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359288716480 : -109714420835136 : ffff9c371eeb5cc0) 00000100:00000001:0.0:1788572474.441677:0:261453:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572474.441680:0:261453:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00000100:00000001:0.0:1788572474.441685:0:261453:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572474.441690:0:261453:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.441692:0:261453:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572474.441694:0:261453:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572474.441708:0:261453:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5880 x1875440809955456/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.441717:0:261453:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572474.441720:0:261453:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.441724:0:261453:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572474.441727:0:261453:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572474.441730:0:261453:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572474.441733:0:261453:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5880 pname:cluuid:pid:xid:nid:opc:job getfattr:5fa795d2-07a0-4260-8bbc-bb3cde572406:261453:1875440809955456:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572474.441738:0:261453:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572474.441744:0:261453:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572474.441747:0:261453:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.441750:0:261453:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572474.441755:0:261453:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109e9000. 02000000:00000001:0.0:1788572474.441757:0:261453:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572474.441768:0:261453:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572474.441772:0:261453:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8aa8. 00000400:00000010:0.0:1788572474.441781:0:261453:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788572474.441792:0:261453:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809955456, portal 10 00000100:00000001:0.0:1788572474.441795:0:261453:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572474.441796:0:261453:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985024 : -109714177566592 : ffff9c372d6b5880) 00000100:00000040:0.0:1788572474.441802:0:261453:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5880 x1875440809955456/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.441810:0:261453:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572474.441813:0:261453:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572474.441816:0:261453:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572474.441819:0:261453:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809955456, offset 0 00000400:00000200:0.0:1788572474.441827:0:261453:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572474.441836:0:261453:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.441845:0:261453:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572474.441851:0:261453:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572474.441856:0:261453:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572474.441861:0:261453:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572474.441864:0:261453:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572474.441869:0:261453:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539844:539844:256:4294967295] 192.168.202.148@tcp LPNI seq info [539809:539809:8:4294967295] 00000400:00000200:0.0:1788572474.441874:0:261453:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.441884:0:261453:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572474.441892:0:261453:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572474.446484:0:261453:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572474.446514:0:261453:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572474.446524:0:261453:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572474.446529:0:261453:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572474.446550:0:261453:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572474.446559:0:261453:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15043 usecs (16 jiffies) 00000400:00000200:0.0:1788572474.446568:0:261453:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572474.446572:0:261453:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307613468 00000100:00000001:0.0:1788572474.446578:0:261453:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.446580:0:261453:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307613468 00000100:00000001:0.0:1788572474.446583:0:261453:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.446586:0:261453:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572474.446591:0:261453:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.446595:0:261453:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572474.446598:0:261453:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572474.446601:0:261453:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371eeb5cc0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572474.446605:0:261453:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572474.446617:0:261453:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572474.461281:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.461314:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.461320:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.461323:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.461335:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.461345:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.461360:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.461366:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.461373:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.461379:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465080 00000400:00000200:0.0:1788572474.461385:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.461393:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a141 [1] + 224 00000800:00000001:0.0:1788572474.461403:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.461501:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.461506:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.461512:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.461523:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.461537:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5880 x1875440809955456/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.461548:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c372d6b5880 x1875440809955456/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.461584:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.461595:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.461599:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.470495:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.470510:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.470524:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.470527:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.470537:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.470545:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.470554:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.470559:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.470565:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.470569:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465100 00000400:00000200:0.0:1788572474.470575:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.470581:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a151 [1] + 224 00000800:00000001:0.0:1788572474.470588:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.470594:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.470598:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.470602:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.470609:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.470618:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70700 x1875440809955584/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.470630:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70700 x1875440809955584/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.470678:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.470687:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.470690:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.496460:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.496465:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572474.496470:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.496475:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.496480:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572474.496483:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572474.496487:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.496491:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4000 x1875440809955712/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.496501:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.496504:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4000 x1875440809955712/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.496512:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.496513:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.507734:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.507753:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.507757:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.507760:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.507769:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.507776:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.507784:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.507790:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.507794:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.507798:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465180 00000400:00000200:0.0:1788572474.507803:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.507809:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a159 [1] + 224 00000800:00000001:0.0:1788572474.507816:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.507823:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.507826:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.507830:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.507848:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.507857:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4000 x1875440809955712/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.507868:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4000 x1875440809955712/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.507891:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.507908:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.507912:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572474.510827:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572474.510830:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572474.519161:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.519171:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98000. 00000400:00000200:0.0:1788572474.519178:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.519186:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.519195:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788572474.519197:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788572474.519203:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.519212:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4000 x1875440809955840/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.519225:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.519229:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4000 x1875440809955840/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.519237:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.519239:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.523137:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.523154:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.523158:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.523160:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.523171:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.523178:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.523187:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.523192:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.523197:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.523201:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465200 00000400:00000200:0.0:1788572474.523206:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.523212:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a169 [1] + 224 00000800:00000001:0.0:1788572474.523219:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.523227:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.523231:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.523235:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.523242:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.523254:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4000 x1875440809955840/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.523265:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c4000 x1875440809955840/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.523286:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.523295:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.523298:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.547465:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.547476:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572474.547482:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.547491:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.547499:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1198 00000400:00000010:0.0:1788572474.547502:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1198. 00000100:00000001:0.0:1788572474.547507:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.547514:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809955968/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.547525:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.547528:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6aa00 x1875440809955968/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.547540:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.547542:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.552283:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.552303:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.552308:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.552311:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.552322:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.552331:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.552340:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.552346:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.552352:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.552356:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465280 00000400:00000200:0.0:1788572474.552374:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.552387:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a179 [1] + 224 00000800:00000001:0.0:1788572474.552395:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.552404:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.552407:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.552412:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.552420:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.552430:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809955968/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.552441:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed6aa00 x1875440809955968/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.552460:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.552468:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.552471:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.554610:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.554614:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572474.554619:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.554624:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.554629:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1220 00000400:00000010:0.0:1788572474.554632:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1220. 00000100:00000001:0.0:1788572474.554657:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.554662:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809956096/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.554671:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.554674:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440809956096/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.554680:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.554681:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.558354:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.558371:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.558375:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.558377:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.558395:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.558403:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.558412:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.558416:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.558421:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.558425:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465300 00000400:00000200:0.0:1788572474.558430:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.558437:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a181 [1] + 224 00000800:00000001:0.0:1788572474.558445:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.558453:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.558456:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.558461:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.558468:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.558477:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809956096/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.558489:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6aa00 x1875440809956096/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.558512:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.558519:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.558523:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.605739:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.605746:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572474.605751:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.605756:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.605762:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1088 00000400:00000010:0.0:1788572474.605764:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1088. 00000100:00000001:0.0:1788572474.605768:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.605772:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69f80 x1875440809956224/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.605781:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.605784:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed69f80 x1875440809956224/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.605790:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.605792:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.621537:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.621555:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.621559:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.621563:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.621572:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.621579:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.621589:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.621594:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.621599:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.621604:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465380 00000400:00000200:0.0:1788572474.621609:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.621616:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a189 [1] + 224 00000800:00000001:0.0:1788572474.621624:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.621748:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.621752:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.621757:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.621764:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.621772:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69f80 x1875440809956224/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.621783:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed69f80 x1875440809956224/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.621801:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.621811:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.621814:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.624314:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.624319:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572474.624323:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.624329:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.624334:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1e58 00000400:00000010:0.0:1788572474.624337:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1e58. 00000100:00000001:0.0:1788572474.624342:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.624346:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69f80 x1875440809956352/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.624355:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.624358:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440809956352/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.624365:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.624366:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.631710:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.631730:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.631734:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.631737:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.631745:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.631754:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.631763:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.631768:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.631774:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.631779:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465400 00000400:00000200:0.0:1788572474.631784:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.631790:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a191 [1] + 224 00000800:00000001:0.0:1788572474.631799:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.631805:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.631809:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.631813:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.631821:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.631830:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69f80 x1875440809956352/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.631842:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed69f80 x1875440809956352/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.631865:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.631875:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.631878:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.666599:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.666605:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572474.666610:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.666616:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.666920:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572474.666924:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572474.666930:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.666937:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4000 x1875440809956480/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.666947:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.666951:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4000 x1875440809956480/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.666958:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.666960:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.677482:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.677501:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.677506:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.677509:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.677518:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.677527:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.677535:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.677541:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.677546:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.677550:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465480 00000400:00000200:0.0:1788572474.677556:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.677561:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a199 [1] + 224 00000800:00000001:0.0:1788572474.677569:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.677576:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.677580:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.677584:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.677592:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.677600:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4000 x1875440809956480/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.677611:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4000 x1875440809956480/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.677666:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.677675:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.677678:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.680822:0:261461:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.680825:0:261461:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.680827:0:261461:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.680830:0:261461:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.680832:0:261461:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.680833:0:261461:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdf2:0x0] 00800000:00000001:0.0:1788572474.680835:0:261461:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.680838:0:261461:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xd94:0x0], fid2=[0x200002b17:0xdf2:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572474.680845:0:261461:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.680850:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215ba70 00000002:00010000:0.0:1788572474.680854:0:261461:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdf2:0x0]) in obj [0x200002b17:0xd94:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572474.680858:0:261461:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572474.680860:0:261461:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572474.680863:0:261461:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572474.680871:0:261461:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3705277700 count: 3 00010000:00000001:0.0:1788572474.680873:0:261461:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.680879:0:261461:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.680883:0:261461:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572474.680888:0:261461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731fe6a00/0x144158609316880b lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdf2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2b43 expref: -99 pid: 261461 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572474.680908:0:261461:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572474.680915:0:261461:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.680918:0:261461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731fe6a00/0x144158609316880b lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdf2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2b43 expref: -99 pid: 261461 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572474.680925:0:261461:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572474.680928:0:261461:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572474.680929:0:261461:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.680931:0:261461:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.680934:0:261461:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.680936:0:261461:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xd94:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572474.680940:0:261461:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572474.680944:0:261461:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c3704c3a040 2 00000080:00000010:0.0:1788572474.680948:0:261461:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711214a00. 00000080:00000001:0.0:1788572474.680954:0:261461:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572474.680955:0:261461:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.680968:0:261461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdf2:0x0](ffff9c37020e0908) 00000080:00010000:0.0:1788572474.680972:0:261461:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdf2:0x0](ffff9c37020e0908) for lock 0x144158609316880b 00800000:00000001:0.0:1788572474.680976:0:261461:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572474.680978:0:261461:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572474.680980:0:261461:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.680982:0:261461:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.680985:0:261461:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe6a00 refcount=3 00000020:00000001:0.0:1788572474.680987:0:261461:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608732160 : -109714100819456 : ffff9c3731fe6a00) 00010000:00000001:0.0:1788572474.680990:0:261461:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608732160 : -109714100819456 : ffff9c3731fe6a00) 00010000:00000001:0.0:1788572474.680993:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.680994:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572474.680996:0:261461:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572474.680998:0:261461:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.681000:0:261461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316880b from it ffffb95bc215bcb8 00010000:00000001:0.0:1788572474.681002:0:261461:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.681003:0:261461:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.681005:0:261461:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe6a00 refcount=3 00000020:00000001:0.0:1788572474.681006:0:261461:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608732160 : -109714100819456 : ffff9c3731fe6a00) 00010000:00000001:0.0:1788572474.681008:0:261461:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608732160 : -109714100819456 : ffff9c3731fe6a00) 00010000:00000001:0.0:1788572474.681010:0:261461:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572474.681013:0:261461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731fe6a00/0x144158609316880b lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdf2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2b43 expref: -99 pid: 261461 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572474.681020:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.681021:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.681022:0:261461:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572474.681024:0:261461:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572474.681026:0:261461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3731fe6a00/0x144158609316880b lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdf2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a2b43 expref: -99 pid: 261461 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572474.681033:0:261461:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572474.681036:0:261461:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572474.681038:0:261461:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.681041:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.681042:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.681044:0:261461:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681048:0:261461:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572474.681049:0:261461:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681051:0:261461:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681054:0:261461:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681056:0:261461:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572474.681057:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.681058:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572474.681061:0:261461:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572474.681062:0:261461:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572474.681065:0:261461:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572474.681068:0:261461:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572474.681071:0:261461:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572474.681073:0:261461:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572474.681075:0:261461:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.681077:0:261461:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572474.681080:0:261461:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572474.681083:0:261461:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572474.681087:0:261461:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634358887902456 : -109714821649160 : ffff9c3707076cf8) 00000020:00001000:0.0:1788572474.681099:0:261461:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3707076cf0 00000020:00000001:0.0:1788572474.681105:0:261461:0:(cl_object.c:143:cl_object_top()) ffff9c3704348468 -> ffff9c3704348468 00000020:00000001:0.0:1788572474.681108:0:261461:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572474.681110:0:261461:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572474.681112:0:261461:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdf2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572474.681115:0:261461:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572474.681117:0:261461:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572474.681119:0:261461:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572474.681121:0:261461:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.681122:0:261461:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572474.681128:0:261461:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdf2:0x0]io ffff9c37313242b0 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572474.681131:0:261461:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cbaaa8) owner 0000000000000000/ffff9c3703aa4f80 00020000:00000001:0.0:1788572474.681135:0:261461:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572474.681137:0:261461:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572474.681139:0:261461:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572474.681141:0:261461:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681143:0:261461:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572474.681145:0:261461:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572474.681146:0:261461:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681148:0:261461:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572474.681148:0:261461:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572474.681149:0:261461:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572474.681151:0:261461:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cbaaa8) owner 0000000000000000/ffff9c3703aa4f80 00000020:00000001:0.0:1788572474.681153:0:261461:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572474.681155:0:261461:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572474.681156:0:261461:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdf2:0x0] 00000020:00000001:0.0:1788572474.681159:0:261461:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572474.681161:0:261461:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572474.681163:0:261461:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572474.681166:0:261461:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572474.681171:0:261461:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572474.681180:0:261461:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676180. 00020000:00000002:0.0:1788572474.681184:0:261461:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572474.681187:0:261461:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572474.681189:0:261461:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572474.681190:0:261461:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572474.681191:0:261461:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572474.681193:0:261461:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186943656 : -109714522607960 : ffff9c3718da6ea8) 00000020:00001000:0.0:1788572474.681195:0:261461:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da6ea0 00000020:00000001:0.0:1788572474.681198:0:261461:0:(cl_object.c:143:cl_object_top()) ffff9c3704348148 -> ffff9c3704348468 00000020:00000001:0.0:1788572474.681200:0:261461:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572474.681203:0:261461:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572474.681204:0:261461:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572474.681205:0:261461:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681207:0:261461:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681209:0:261461:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314383536 : -109714395168080 : ffff9c37207302b0) 00000020:00000001:0.0:1788572474.681210:0:261461:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572474.681215:0:261461:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6d10. 00000008:00010000:0.0:1788572474.681219:0:261461:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572474.681221:0:261461:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572474.681223:0:261461:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572474.681225:0:261461:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d6761b0, osc lock ffff9c37030a6d10, flags 0x400001002 00000020:00000001:0.0:1788572474.681226:0:261461:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681228:0:261461:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681231:0:261461:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531725184 : -109714177826432 : ffff9c372d676180) 00000020:00000001:0.0:1788572474.681233:0:261461:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572474.681234:0:261461:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572474.681236:0:261461:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681237:0:261461:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681240:0:261461:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572474.681242:0:261461:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572474.681243:0:261461:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572474.681245:0:261461:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572474.681246:0:261461:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572474.681247:0:261461:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314383536 : -109714395168080 : ffff9c37207302b0) 00000020:00000001:0.0:1788572474.681249:0:261461:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572474.681251:0:261461:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572474.681253:0:261461:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572474.681256:0:261461:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572474.681258:0:261461:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572474.681260:0:261461:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572474.681262:0:261461:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572474.681263:0:261461:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572474.681264:0:261461:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572474.681266:0:261461:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572474.681268:0:261461:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572474.681272:0:261461:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681274:0:261461:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572474.681279:0:261461:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572474.681283:0:261461:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572474.681287:0:261461:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572474.681290:0:261461:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.681293:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.681294:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.681296:0:261461:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.681299:0:261461:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572474.681301:0:261461:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572474.681304:0:261461:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572474.681310:0:261461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37101b6600. 00000100:00000001:0.0:1788572474.681317:0:261461:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681322:0:261461:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572474.681324:0:261461:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681325:0:261461:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572474.681326:0:261461:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681328:0:261461:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.681331:0:261461:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00010000:00000001:0.0:1788572474.681334:0:261461:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572474.681337:0:261461:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572474.681341:0:261461:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3710a87b00. 00010000:00000010:0.0:1788572474.681344:0:261461:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3710a87400. 00010000:00000001:0.0:1788572474.681348:0:261461:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572474.681350:0:261461:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788572474.681354:0:261461:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572474.681357:0:261461:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168812 to hash 00000020:00000001:0.0:1788572474.681359:0:261461:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572474.681360:0:261461:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000010:0.0:1788572474.681364:0:261461:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372d091d00. 00010000:00000001:0.0:1788572474.681367:0:261461:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788572474.681370:0:261461:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.681371:0:261461:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.681375:0:261461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000db0f8a34/0x1441586093168812 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x528a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261461 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572474.681384:0:261461:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000db0f8a34/0x1441586093168812 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x528a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 261461 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572474.681394:0:261461:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.681398:0:261461:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572474.681402:0:261461:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572474.681414:0:261461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5f80 x1875440809956608/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572474.681452:0:261461:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572474.681455:0:261461:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681457:0:261461:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.681459:0:261461:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681461:0:261461:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.681463:0:261461:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572474.681465:0:261461:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572474.681466:0:261461:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572474.681468:0:261461:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572474.681476:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572474.681480:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572474.681482:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572474.681487:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572474.681489:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572474.681492:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572474.681493:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572474.681495:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572474.681502:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809956608/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.681509:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572474.681512:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.681516:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572474.681518:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572474.681521:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572474.681524:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809956608:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572474.681529:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572474.681534:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572474.681536:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572474.681538:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572474.681543:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788572474.681545:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572474.681552:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572474.681556:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8cb8. 00000400:00000010:0.0:1788572474.681564:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572474.681570:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809956608, portal 4 00000100:00000001:0.0:1788572474.681573:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572474.681574:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572474.681580:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5f80 x1875440809956608/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.681587:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572474.681590:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572474.681592:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572474.681594:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 28, xid 1875440809956608, offset 0 00000400:00000200:0.0:1788572474.681600:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572474.681609:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.681615:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572474.681620:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572474.681625:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572474.681628:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572474.681630:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572474.682051:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539853:539853:256:4294967295] 192.168.202.148@tcp LPNI seq info [539818:539818:8:4294967295] 00000400:00000200:0.0:1788572474.682058:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.682068:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572474.682075:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572474.682081:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572474.682148:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572474.682156:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572474.682161:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572474.682178:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572474.682183:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 165507 usecs (166 jiffies) 00000400:00000200:0.0:1788572474.682186:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572474.682189:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307613854 00000100:00000001:0.0:1788572474.682192:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.682194:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307613854 00000100:00000001:0.0:1788572474.682196:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307614439 00000100:00000001:0.0:1788572474.682198:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572474.682201:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.682204:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.682207:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572474.689494:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.689521:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.689525:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.689527:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.689537:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.689544:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.689554:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.689559:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.689564:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.689568:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465500 00000400:00000200:0.0:1788572474.689573:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.689579:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a1a9 [1] + 224 00000800:00000001:0.0:1788572474.689587:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.689599:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.689605:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.689609:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.689617:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.689627:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5f80 x1875440809956608/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.689682:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c372d6b5f80 x1875440809956608/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.689700:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.689708:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.689712:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572474.690753:0:261461:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.690757:0:261461:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.690764:0:261461:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572474.690766:0:261461:0:(cl_object.c:143:cl_object_top()) ffff9c3704348468 -> ffff9c3704348468 00000020:00000001:0.0:1788572474.690769:0:261461:0:(cl_object.c:143:cl_object_top()) ffff9c3704348468 -> ffff9c3704348468 00000020:00000001:0.0:1788572474.690771:0:261461:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572474.690775:0:261461:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572474.690779:0:261461:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572474.690781:0:261461:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572474.690783:0:261461:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572474.690784:0:261461:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572474.690786:0:261461:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdf2:11031 on OST[0]: s=0 (1) m=1788572469 a=1788572469 c=1788572469 b=0 00020000:00000001:0.0:1788572474.690789:0:261461:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572474.690792:0:261461:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572469 a=1788572469 c=1788572469 b=0 00020000:00000001:0.0:1788572474.690794:0:261461:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.690796:0:261461:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.690798:0:261461:0:(cl_object.c:143:cl_object_top()) ffff9c3704348468 -> ffff9c3704348468 00000080:00200000:0.0:1788572474.690800:0:261461:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdf2:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572474.690803:0:261461:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572474.690806:0:261461:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572474.690808:0:261461:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572474.690811:0:261461:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572474.690812:0:261461:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572474.690815:0:261461:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572474.690819:0:261461:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572474.690820:0:261461:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359314383536 : -109714395168080 : ffff9c37207302b0) 00000020:00000001:0.0:1788572474.690822:0:261461:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572474.690824:0:261461:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c372d6761b0 (ffff9c3718da6ea8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572474.690826:0:261461:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572474.690827:0:261461:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572474.690829:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.690830:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572474.690831:0:261461:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572474.690841:0:261461:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.690842:0:261461:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.690843:0:261461:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572474.690845:0:261461:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572474.690846:0:261461:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c3731324250 (ffff9c3707076cf8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572474.690848:0:261461:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572474.690850:0:261461:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572474.690851:0:261461:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c372d6761b0 (ffff9c3707076cf8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572474.690855:0:261461:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c37030a6d10. 00000020:00000001:0.0:1788572474.690858:0:261461:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572474.690859:0:261461:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c372d676180. 00020000:00000001:0.0:1788572474.690866:0:261461:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572474.690867:0:261461:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572474.690868:0:261461:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572474.690870:0:261461:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.690873:0:261461:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572474.690876:0:261461:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572474.690877:0:261461:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572474.690879:0:261461:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572474.690880:0:261461:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572474.690882:0:261461:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da6ea0 00020000:00000001:0.0:1788572474.690886:0:261461:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572474.690887:0:261461:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572474.690890:0:261461:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572474.690892:0:261461:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdf2:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572474.690904:0:261461:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572474.690906:0:261461:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.690909:0:261461:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572474.690911:0:261461:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572474.690912:0:261461:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572474.690913:0:261461:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3707076cf0 00000001:00000001:0.0:1788572474.690915:0:261461:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.690923:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bd08 00000080:00000008:0.0:1788572474.690926:0:261461:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f434.sanity-094:[0x200002b17:0xdf2:0x0](ffff9c37020e0908), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572474.690932:0:261461:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572474.690934:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bda0 00000080:00002000:0.0:1788572474.690936:0:261461:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-094 (ffff9c37035995e8, parent ffff9c3719d93af8, inode ffff9c37020e0908) hashed, 00000080:00000001:0.0:1788572474.690938:0:261461:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.690979:0:261461:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.690983:0:261461:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572474.690985:0:261461:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572474.690986:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bbc0 00000080:00200000:0.0:1788572474.690988:0:261461:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572474.690994:0:261461:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7000. 00800000:00000002:0.0:1788572474.691000:0:261461:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572474.691004:0:261461:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572474.691008:0:261461:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.691010:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bae0 00800000:00000002:0.0:1788572474.691012:0:261461:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.691015:0:261461:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572474.691018:0:261461:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.691020:0:261461:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.691023:0:261461:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.691025:0:261461:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.691027:0:261461:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.691029:0:261461:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.691032:0:261461:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.691035:0:261461:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572474.691040:0:261461:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.691042:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215b9b0 00000002:00010000:0.0:1788572474.691043:0:261461:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572474.691046:0:261461:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572474.691048:0:261461:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572474.691050:0:261461:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572474.691057:0:261461:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572474.691060:0:261461:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.691065:0:261461:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.691068:0:261461:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572474.691071:0:261461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572474.691080:0:261461:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572474.691084:0:261461:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.691087:0:261461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572474.691298:0:261461:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572474.691300:0:261461:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572474.691302:0:261461:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.691304:0:261461:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.691307:0:261461:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.691309:0:261461:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572474.691311:0:261461:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572474.691315:0:261461:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572474.691318:0:261461:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7000. 00000080:00000001:0.0:1788572474.691325:0:261461:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572474.691326:0:261461:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.691329:0:261461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572474.691332:0:261461:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572474.691336:0:261461:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572474.691338:0:261461:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572474.691339:0:261461:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.691341:0:261461:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.691343:0:261461:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.691345:0:261461:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.691347:0:261461:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.691349:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.691350:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572474.691351:0:261461:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572474.691353:0:261461:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.691356:0:261461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc215bbf8 00010000:00000001:0.0:1788572474.691358:0:261461:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.691359:0:261461:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.691360:0:261461:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.691362:0:261461:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.691364:0:261461:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.691365:0:261461:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572474.691368:0:261461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.691374:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.691375:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.691376:0:261461:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572474.691378:0:261461:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572474.691380:0:261461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.691387:0:261461:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572474.691388:0:261461:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572474.691390:0:261461:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.691392:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.691393:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.691395:0:261461:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.691397:0:261461:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572474.691398:0:261461:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.691400:0:261461:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.691403:0:261461:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.691404:0:261461:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572474.691405:0:261461:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.691407:0:261461:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572474.691410:0:261461:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572474.691411:0:261461:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.691414:0:261461:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572474.691418:0:261461:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.691422:0:261461:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.691425:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bb88 00000080:00002000:0.0:1788572474.691426:0:261461:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 94 00000080:00000001:0.0:1788572474.691429:0:261461:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.691431:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bbd0 00000080:00200000:0.0:1788572474.691432:0:261461:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572474.691437:0:261461:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572474.691441:0:261461:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572474.691443:0:261461:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572474.691445:0:261461:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.691448:0:261461:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.691449:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bbe8 00000080:00002000:0.0:1788572474.691451:0:261461:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-094(ffff9c37035995e8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572474.691453:0:261461:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.691454:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215bc30 00000080:00200000:0.0:1788572474.691456:0:261461:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-094, flags=9 00000080:00000001:0.0:1788572474.691467:0:261461:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572474.691470:0:261461:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572474.691473:0:261461:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572474.691474:0:261461:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc215be40 00000080:00002000:0.0:1788572474.691475:0:261461:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-094 (ffff9c37035995e8, parent ffff9c3719d93af8, inode ffff9c37020e0908) hashed, 00000080:00000001:0.0:1788572474.691478:0:261461:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572474.735573:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.735578:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572474.735583:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.735589:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.735594:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788572474.735597:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788572474.735601:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.735605:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69f80 x1875440809956736/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.735614:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.735618:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed69f80 x1875440809956736/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.735624:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.735626:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.743591:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.743609:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.743613:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.743616:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.743625:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.743668:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.743678:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.743683:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.743688:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.743693:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465580 00000400:00000200:0.0:1788572474.743698:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.743704:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a1b9 [1] + 224 00000800:00000001:0.0:1788572474.743711:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.743763:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.743768:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.743773:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.743781:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.743790:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69f80 x1875440809956736/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.743801:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371ed69f80 x1875440809956736/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.743812:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.743819:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.743823:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.785184:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000080:00000010:0.0:1788572474.913255:0:261467:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c37101b7200. 00800000:00000002:0.0:1788572474.913262:0:261467:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572474.913267:0:261467:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572474.913271:0:261467:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.913273:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153ae0 00800000:00000002:0.0:1788572474.913275:0:261467:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.913279:0:261467:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572474.913282:0:261467:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572474.913284:0:261467:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572474.913286:0:261467:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572474.913288:0:261467:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572474.913290:0:261467:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.913292:0:261467:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572474.913294:0:261467:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572474.913297:0:261467:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572474.913303:0:261467:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572474.913304:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc21539b0 00000002:00010000:0.0:1788572474.913306:0:261467:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572474.913309:0:261467:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572474.913310:0:261467:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572474.913313:0:261467:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572474.913319:0:261467:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572474.913322:0:261467:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572474.913326:0:261467:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572474.913329:0:261467:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572474.913333:0:261467:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572474.913340:0:261467:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572474.913345:0:261467:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572474.913348:0:261467:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00000001:0.0:1788572474.913354:0:261467:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572474.913356:0:261467:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572474.913358:0:261467:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.913359:0:261467:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572474.913361:0:261467:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.913363:0:261467:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572474.913365:0:261467:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572474.913368:0:261467:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572474.913371:0:261467:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c37101b7200. 00000080:00000001:0.0:1788572474.913377:0:261467:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572474.913379:0:261467:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.913381:0:261467:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572474.913384:0:261467:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093167cc0 00800000:00000001:0.0:1788572474.913545:0:261467:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572474.913548:0:261467:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572474.913550:0:261467:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.913552:0:261467:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.913554:0:261467:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.913557:0:261467:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.913559:0:261467:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.913562:0:261467:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.913564:0:261467:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572474.913565:0:261467:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572474.913567:0:261467:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572474.913570:0:261467:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093167cc0 from it ffffb95bc2153bf8 00010000:00000001:0.0:1788572474.913572:0:261467:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572474.913573:0:261467:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572474.913574:0:261467:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c37421cb600 refcount=3 00000020:00000001:0.0:1788572474.913576:0:261467:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.913578:0:261467:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359879153152 : -109713830398464 : ffff9c37421cb600) 00010000:00000001:0.0:1788572474.913580:0:261467:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572474.913583:0:261467:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.913590:0:261467:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.913592:0:261467:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.913593:0:261467:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572474.913595:0:261467:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572474.913597:0:261467:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00000001:0.0:1788572474.913604:0:261467:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572474.913606:0:261467:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572474.913607:0:261467:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572474.913610:0:261467:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.913611:0:261467:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572474.913612:0:261467:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.913615:0:261467:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572474.913616:0:261467:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.913618:0:261467:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.913620:0:261467:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572474.913622:0:261467:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572474.913624:0:261467:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572474.913625:0:261467:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572474.913627:0:261467:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572474.913628:0:261467:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572474.913630:0:261467:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572474.913662:0:261467:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.913667:0:261467:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.913669:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153b88 00000080:00002000:0.0:1788572474.913671:0:261467:0:(dcache.c:87:ll_dcompare()) found name d434.sanity(ffff9c3719d93af8) flags 0x28004e refc 97 00000080:00000001:0.0:1788572474.913674:0:261467:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.913676:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153bd0 00000080:00200000:0.0:1788572474.913678:0:261467:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d434.sanity, flags=25 00000080:00000002:0.0:1788572474.913683:0:261467:0:(llite_lib.c:3288:ll_update_dir_depth_dmv()) d434.sanity [0x200002b17:0xd94:0x0] depth 1 server default LMV inherit depth 0 00000080:00000001:0.0:1788572474.913687:0:261467:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572474.913689:0:261467:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xd94:0x0](ffff9c37087b9148), inode mode 41ed mask 1 00000080:00000001:0.0:1788572474.913691:0:261467:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572474.913694:0:261467:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572474.913695:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153be8 00000080:00002000:0.0:1788572474.913697:0:261467:0:(dcache.c:87:ll_dcompare()) found name f434.sanity-097(ffff9c370615aa20) flags 0x48000e refc 0 00000080:00000001:0.0:1788572474.913700:0:261467:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572474.913701:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153c30 00000080:00200000:0.0:1788572474.913703:0:261467:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f434.sanity-097, flags=9 00000080:00000001:0.0:1788572474.913714:0:261467:0:(xattr.c:558:ll_xattr_get_common()) Process entered 00000080:00000001:0.0:1788572474.913717:0:261467:0:(xattr.c:565:ll_xattr_get_common()) Process leaving (rc=18446744073709551521 : -95 : ffffffffffffffa1) 00000080:00000001:0.0:1788572474.913720:0:261467:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572474.913721:0:261467:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2153e40 00000080:00002000:0.0:1788572474.913723:0:261467:0:(dcache.c:127:ll_ddelete()) keeping dentry f434.sanity-097 (ffff9c370615aa20, parent ffff9c3719d93af8, inode ffff9c37020e1148) hashed, 00000080:00000001:0.0:1788572474.913726:0:261467:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572474.967812:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.967843:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.967847:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.967851:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.967860:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.967868:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.967877:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.967882:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.967888:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.967893:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465880 00000400:00000200:0.0:1788572474.967908:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.967914:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a1e9 [1] + 224 00000800:00000001:0.0:1788572474.967922:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.967991:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.967995:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.968001:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.968009:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.968019:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4000 x1875440809957504/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.968030:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c4000 x1875440809957504/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572559 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.968066:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.968074:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.968077:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572474.971303:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572474.971309:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3720ff3900. 00000400:00000200:0.0:1788572474.971315:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.971322:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572474.971328:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1908 00000400:00000010:0.0:1788572474.971330:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1908. 00000100:00000001:0.0:1788572474.971335:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572474.971341:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71c00 x1875440809957632/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.971351:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572474.971353:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440809957632/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.971360:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572474.971362:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572474.976773:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.976792:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572474.976796:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.976799:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.976807:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.976816:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572474.976826:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.976831:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.976845:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.976849:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417465900 00000400:00000200:0.0:1788572474.976855:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572474.976861:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a1f9 [1] + 224 00000800:00000001:0.0:1788572474.976868:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.976876:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572474.976880:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572474.976885:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572474.976892:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572474.976910:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71c00 x1875440809957632/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572474.976920:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec71c00 x1875440809957632/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572490 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572474.976940:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572474.976948:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572474.976951:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572475.002916:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572475.002921:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572475.002927:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572475.002934:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572475.002940:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788572475.002942:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788572475.002946:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572475.002951:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5180 x1875440809957760/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.002971:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572475.002975:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5180 x1875440809957760/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.002983:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.002984:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572475.008829:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572475.008850:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572475.008854:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572475.008857:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572475.008865:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572475.008882:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572475.008893:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572475.008906:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572475.008912:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572475.008916:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 640 into portal 10 MB=0x6a9b417465980 00000400:00000200:0.0:1788572475.008922:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572475.008929:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 640/640 into md 0x52a201 [1] + 224 00000800:00000001:0.0:1788572475.008937:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572475.008947:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572475.008951:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572475.008968:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572475.008977:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572475.008987:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5180 x1875440809957760/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572475.008998:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=640 offset=224 replen=1680 req@ffff9c371f1c5180 x1875440809957760/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.009019:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572475.011310:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572475.011315:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572475.011622:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572475.011628:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98d00. 00000400:00000200:0.0:1788572475.011633:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572475.011672:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572475.011679:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572475.011681:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572475.011685:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572475.011691:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.011700:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572475.011703:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.011708:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.011709:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572475.019953:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572475.019973:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572475.019980:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572475.019982:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572475.019986:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2aa8. 00000400:00000200:0.0:1788572475.019992:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572475.019995:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572475.019998:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572475.020004:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572475.020014:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.020021:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572475.020024:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572475.020026:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572475.020030:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572475.020032:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572475.020035:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572475.020038:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572475.020048:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=39 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572475.020052:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=12 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572475.020054:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572475.020056:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020059:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572475.020063:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.020066:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572475.020069:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010070 00000100:00000001:0.0:1788572475.020071:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572475.020074:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572475.020079:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.020086:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572475.020089:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572475.020091:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572475.020094:0:2404:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572475.020096:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020099:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020102:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=4 00000020:00000001:0.0:1788572475.020104:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020107:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020109:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020110:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020111:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=5 00000020:00000001:0.0:1788572475.020112:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020114:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020117:0:2404:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572475.020121:0:2404:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572475.020125:0:2404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020141:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020143:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572475.020145:0:2404:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572475.020148:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020149:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020150:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=6 00000020:00000001:0.0:1788572475.020152:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020154:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000040:0.0:1788572475.020161:0:2404:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c3731fe7a00, remote cookie: 0xd9894672d99a2bf9, flags: 0x1001 00010000:00010000:0.0:1788572475.020166:0:2404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020173:0:2404:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572475.020176:0:2404:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020178:0:2404:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572475.020182:0:2404:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572475.020186:0:2404:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572475.020194:0:2404:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x2c0000401:0x521a:0x0].0x0 (ffff9c372067a200) refcount = 1 00010000:00000040:0.0:1788572475.020197:0:2404:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572475.020199:0:2404:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020208:0:2404:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572475.020210:0:2404:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572475.020213:0:2404:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572475.020215:0:2404:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572475.020219:0:2404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020226:0:2404:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572475.020227:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020228:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572475.020230:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020231:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572475.020233:0:2404:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572475.020236:0:2404:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572475.020238:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020240:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020241:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=5 00000020:00000001:0.0:1788572475.020244:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020246:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00000008:00000001:0.0:1788572475.020250:0:2404:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572475.020254:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000008:00010000:0.0:1788572475.020259:0:2404:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572475.020266:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000020:00000001:0.0:1788572475.020268:0:2404:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572475.020270:0:2404:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572475.020274:0:2404:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572475.020276:0:2404:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572475.020278:0:2404:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572475.020280:0:2404:0:(cl_object.c:143:cl_object_top()) ffff9c3720894540 -> ffff9c3718eebd00 00000008:00000001:0.0:1788572475.020282:0:2404:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572475.020284:0:2404:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572475.020286:0:2404:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572475.020305:0:2404:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572475.020308:0:2404:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020310:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020311:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020313:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=6 00000020:00000001:0.0:1788572475.020315:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020318:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020320:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572475.020323:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020330:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020331:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572475.020334:0:2404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020341:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572475.020342:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020343:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572475.020344:0:2404:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020346:0:2404:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572475.020347:0:2404:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572475.020348:0:2404:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3731fe7a00 refcount=5 00000020:00000001:0.0:1788572475.020350:0:2404:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020352:0:2404:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359608736256 : -109714100815360 : ffff9c3731fe7a00) 00010000:00000001:0.0:1788572475.020354:0:2404:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572475.020356:0:2404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572475.020362:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020363:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572475.020364:0:2404:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572475.020366:0:2404:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572475.020369:0:2404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9c3711155800 lock: 00000000c93abcc7/0x1441586093168858 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x521a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a2bf9 expref: -99 pid: 261471 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572475.020375:0:2404:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572475.020378:0:2404:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572475.020380:0:2404:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572475.020384:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020385:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572475.020387:0:2404:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020389:0:2404:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572475.020391:0:2404:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020393:0:2404:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020395:0:2404:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572475.020397:0:2404:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572475.020398:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020399:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572475.020400:0:2404:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572475.020401:0:2404:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572475.020402:0:2404:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572475.020408:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572475.020416:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809957888:192.168.202.148@tcp:101:getfattr.0 00000100:00000001:0.0:1788572475.020421:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572475.020424:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572475.020427:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440809957888/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.020434:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572475.020436:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572475.020440:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371fe95c00. 02000000:00000001:0.0:1788572475.020445:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572475.020448:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572475.020450:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=5 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572475.020452:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572475.020454:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711215e00. 02000000:00000001:0.0:1788572475.020459:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572475.020461:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572475.020463:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788572475.020466:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572475.020467:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572475.020470:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572475.020476:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572475.020478:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.020480:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572475.020482:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572475.044951:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572475.044955:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3719a98300. 00000400:00000200:0.0:1788572475.044959:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572475.044964:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572475.044970:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3dd0 00000400:00000010:0.0:1788572475.044972:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3dd0. 00000100:00000001:0.0:1788572475.044976:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572475.044980:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5500 x1875440809958016/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.044990:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572475.044992:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5500 x1875440809958016/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572560 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.044998:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.044999:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572475.065339:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572475.065344:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9900. 00000400:00000200:0.0:1788572475.065350:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572475.065355:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572475.065360:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1f68 00000400:00000010:0.0:1788572475.065363:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1f68. 00000100:00000001:0.0:1788572475.065366:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572475.065370:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71c00 x1875440809958144/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.065378:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572475.065381:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440809958144/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572491 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'getfattr.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572475.065387:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572475.065389:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572475.457733:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307614439, at 4307614464 00000100:00000010:0.0:1788572475.457748:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788572475.585926:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572475.585929:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572475.585932:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572476.740258:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572476.740263:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572476.740269:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.740275:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572476.740280:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1440 00000400:00000010:0.0:1788572476.740283:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1440. 00000100:00000001:0.0:1788572476.740287:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572476.740297:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a300 x1875440809958400/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.740306:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572476.740309:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440809958400/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572561 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.740315:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572476.740317:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572476.740379:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572476.740382:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572476.740385:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.740388:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572476.740391:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1ee0 00000400:00000010:0.0:1788572476.740393:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1ee0. 00000100:00000001:0.0:1788572476.740396:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572476.740399:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69180 x1875440809958656/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.740405:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572476.740408:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440809958656/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.740414:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572476.740416:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572476.777950:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.780730:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572476.780739:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.780745:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.780754:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.780762:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572476.780770:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.780775:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.780780:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.780785:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417465b80 00000400:00000200:0.0:1788572476.780790:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.780796:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a249 [1] + 224 00000800:00000001:0.0:1788572476.780805:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.780809:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.780812:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.780817:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572476.780824:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572476.780834:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809958272/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572476.780845:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6aa00 x1875440809958272/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.781374:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572476.781387:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781393:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572476.781396:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781399:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781403:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781407:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572476.781412:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781415:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781419:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781422:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417465d00 00000400:00000200:0.0:1788572476.781427:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781432:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a241 [1] + 224 00000800:00000001:0.0:1788572476.781437:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781441:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781444:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.781448:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572476.781452:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572476.781457:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69180 x1875440809958656/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572476.781467:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69180 x1875440809958656/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.781476:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572476.781479:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781483:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572476.781485:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781487:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781490:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781494:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572476.781498:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781501:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781503:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781506:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417465c00 00000400:00000200:0.0:1788572476.781510:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781515:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a239 [1] + 224 00000800:00000001:0.0:1788572476.781519:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781523:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781525:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.781528:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572476.781531:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572476.781535:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a300 x1875440809958400/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572561 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572476.781550:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6a300 x1875440809958400/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572561 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.781557:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572476.781561:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781565:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572476.781566:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781568:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781571:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781574:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572476.781578:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781580:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781583:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781585:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417465d80 00000400:00000200:0.0:1788572476.781588:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572476.781591:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a259 [1] + 224 00000800:00000001:0.0:1788572476.781595:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781603:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572476.781606:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572476.781609:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572476.781612:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572476.781616:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809958784/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572476.781624:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6a680 x1875440809958784/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572476.781630:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572476.781633:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572476.781662:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572479.291039:0:261703:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572480.577851:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307619554, at 4307619584 00000100:00000010:0.0:1788572480.577864:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6ae0. 00000020:00000001:0.0:1788572480.706141:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572480.706144:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572480.706147:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572481.866629:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572481.866632:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572481.866714:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572481.866718:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37310d2600. 00000400:00000200:0.0:1788572481.866723:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572481.866729:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572481.866734:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572481.866737:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572481.866742:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572481.866747:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4700 x1875440809958912/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.866768:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.866772:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4700 x1875440809958912/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.866779:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.866780:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572481.866799:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572481.866802:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37310d2300. 00000400:00000200:0.0:1788572481.866805:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572481.866808:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572481.866811:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3880 00000400:00000010:0.0:1788572481.866813:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3880. 00000100:00000001:0.0:1788572481.866815:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572481.866818:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.866825:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.866828:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.866835:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.866836:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572481.866843:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572481.866845:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37310d2c00. 00000400:00000200:0.0:1788572481.866848:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572481.866851:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572481.866854:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3990 00000400:00000010:0.0:1788572481.866855:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3990. 00000100:00000001:0.0:1788572481.866857:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572481.866860:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.866867:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.866870:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.867004:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.867006:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572481.867018:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572481.867020:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2800. 00000400:00000200:0.0:1788572481.867025:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572481.867028:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572481.867031:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3aa0 00000400:00000010:0.0:1788572481.867032:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3aa0. 00000100:00000001:0.0:1788572481.867034:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572481.867038:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.867045:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.867048:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.867054:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.867055:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572481.867241:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572481.867244:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2600. 00000400:00000200:0.0:1788572481.867248:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572481.867251:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572481.867254:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f36e8 00000400:00000010:0.0:1788572481.867256:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f36e8. 00000100:00000001:0.0:1788572481.867259:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572481.867263:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.867270:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.867273:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.867278:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.867280:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572481.873503:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572481.873509:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572481.874203:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572481.874206:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572481.874210:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f26e0. 00000400:00000200:0.0:1788572481.874215:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572481.874217:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572481.874220:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572481.874226:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572481.874236:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874242:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572481.874244:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874247:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572481.874251:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572481.874253:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572481.874255:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572481.874259:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572481.874269:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=45 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572481.874274:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=106 v=5 (1 1 0 0) 00000100:00000001:0.0:1788572481.874277:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572481.874279:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572481.874282:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572481.874286:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.874292:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572481.874295:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869935 00000100:00000001:0.0:1788572481.874297:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572481.874299:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572481.874304:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874312:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572481.874314:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572481.874317:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572481.874320:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572481.874327:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809959168:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572481.874332:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572481.874334:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572481.874335:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2c08. 00000400:00000200:0.0:1788572481.874337:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572481.874339:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572481.874341:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572481.874343:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572481.874350:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874355:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572481.874357:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874358:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572481.874359:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572481.874361:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572481.874362:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572481.874364:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572481.874367:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=45 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572481.874370:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=18 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572481.874373:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572481.874374:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572481.874376:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572481.874378:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.874380:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572481.874381:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010073 00000100:00000001:0.0:1788572481.874383:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572481.874384:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572481.874388:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874394:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572481.874396:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572481.874398:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572481.874401:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572481.874408:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809959424:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572481.874412:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.874415:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.874418:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440809959168/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874425:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572481.874427:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572481.874431:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3713c8d400. 02000000:00000001:0.0:1788572481.874435:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572481.874438:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572481.874440:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572481.874442:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572481.874445:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37310d2d00. 02000000:00000001:0.0:1788572481.874449:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572481.874452:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572481.874454:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788572481.874457:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572481.874459:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874461:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572481.874465:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440809959424/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874471:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572481.874473:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572481.874474:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3713c8d800. 02000000:00000001:0.0:1788572481.874476:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572481.874477:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572481.874479:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572481.874481:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572481.874482:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37310d2400. 02000000:00000001:0.0:1788572481.874484:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572481.874485:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572481.874486:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788572481.874488:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572481.874490:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874493:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.874496:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572481.874498:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572481.874500:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572481.874502:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572481.874503:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f20b0. 00000400:00000200:0.0:1788572481.874505:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788572481.874507:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788572481.874509:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572481.874513:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572481.874522:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874528:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572481.874530:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874531:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572481.874532:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572481.874534:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572481.874535:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572481.874537:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572481.874541:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=45 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572481.874544:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=77 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572481.874546:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572481.874548:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572481.874549:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572481.874551:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.874553:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572481.874555:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572481.874556:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572481.874557:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572481.874560:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874565:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572481.874567:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572481.874568:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572481.874571:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440809959040/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572566 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572481.874578:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809959040:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572481.874582:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572481.874583:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572481.874585:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2c60. 00000400:00000200:0.0:1788572481.874587:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572481.874589:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572481.874591:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572481.874595:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572481.874603:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.874610:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572481.874612:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572481.874613:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572481.874615:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572481.874616:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572481.874618:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572481.874620:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572481.874624:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=45 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572481.874627:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=96 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572481.874629:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572481.874631:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572481.875122:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572481.875126:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572481.875195:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572481.875198:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572481.875200:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572481.875201:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572481.875208:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572481.875215:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572481.875217:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572481.875218:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572481.875222:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7b80 x1875440809959296/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572497 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572481.875229:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809959296:192.168.202.148@tcp:400:kworker.0 00000100:00000040:0.0:1788572485.698044:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307624677, at 4307624704 00000100:00000010:0.0:1788572485.698065:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788572485.826018:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572485.826021:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572485.826024:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1788572486.031422:0:262152:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572486.983838:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572486.983847:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6da00. 00000400:00000200:0.0:1788572486.983853:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.983861:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572486.983867:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1ee0 00000400:00000010:0.0:1788572486.983870:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1ee0. 00000100:00000001:0.0:1788572486.983876:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572486.983883:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71880 x1875440809959552/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.983895:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572486.983898:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71880 x1875440809959552/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984245:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572486.984247:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572486.984266:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572486.984269:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6df00. 00000400:00000200:0.0:1788572486.984272:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.984276:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572486.984279:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c16e8 00000400:00000010:0.0:1788572486.984281:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c16e8. 00000100:00000001:0.0:1788572486.984284:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572486.984288:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73480 x1875440809959808/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984295:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572486.984298:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440809959808/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984304:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572486.984306:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572486.984314:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572486.984317:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d000. 00000400:00000200:0.0:1788572486.984320:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.984323:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572486.984325:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1088 00000400:00000010:0.0:1788572486.984327:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1088. 00000100:00000001:0.0:1788572486.984330:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572486.984334:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72d80 x1875440809960064/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984341:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572486.984345:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72d80 x1875440809960064/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984351:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572486.984352:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572486.984360:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572486.984362:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d300. 00000400:00000200:0.0:1788572486.984365:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.984368:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572486.984370:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1000 00000400:00000010:0.0:1788572486.984372:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1000. 00000100:00000001:0.0:1788572486.984374:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572486.984378:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70380 x1875440809959680/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984385:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572486.984388:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440809959680/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.984393:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572486.984395:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572486.986007:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572486.986014:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6db00. 00000400:00000200:0.0:1788572486.986020:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.986028:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572486.986033:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1198 00000400:00000010:0.0:1788572486.986035:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1198. 00000100:00000001:0.0:1788572486.986040:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572486.986047:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72300 x1875440809959936/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.986057:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572486.986060:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440809959936/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.986066:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572486.986067:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572486.988282:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988302:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.988306:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988309:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.988318:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988325:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572486.988335:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988339:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988344:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988349:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417466180 00000400:00000200:0.0:1788572486.988354:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988360:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2b9 [1] + 224 00000800:00000001:0.0:1788572486.988367:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988371:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.988374:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.988378:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572486.988386:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572486.988395:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73480 x1875440809959808/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572486.988405:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73480 x1875440809959808/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.988427:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572486.988434:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988438:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.988439:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988441:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.988445:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988448:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572486.988453:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988455:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988458:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988461:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417466100 00000400:00000200:0.0:1788572486.988465:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.988469:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2c9 [1] + 224 00000800:00000001:0.0:1788572486.988473:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988479:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.988481:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.988484:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572486.988487:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572486.988491:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70380 x1875440809959680/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572486.988499:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec70380 x1875440809959680/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572571 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.988507:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572486.988510:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.988513:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.992102:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.992124:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.992128:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.992131:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.992139:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.992148:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572486.992156:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993015:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993022:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993027:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417466080 00000400:00000200:0.0:1788572486.993032:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993039:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2b1 [1] + 224 00000800:00000001:0.0:1788572486.993047:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993055:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.993059:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.993063:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572486.993071:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572486.993080:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71880 x1875440809959552/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572486.993098:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec71880 x1875440809959552/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.993121:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572486.993131:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993135:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.993137:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993139:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.993143:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993147:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572486.993153:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993155:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993495:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993500:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417466280 00000400:00000200:0.0:1788572486.993504:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993509:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2c1 [1] + 224 00000800:00000001:0.0:1788572486.993513:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993518:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.993520:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.993523:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572486.993527:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572486.993531:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72d80 x1875440809960064/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572486.993539:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72d80 x1875440809960064/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.993554:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572486.993560:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993564:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572486.993565:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993567:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.993570:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993574:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572486.993579:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993582:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993585:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993587:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417466200 00000400:00000200:0.0:1788572486.993591:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572486.993595:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2d1 [1] + 224 00000800:00000001:0.0:1788572486.993598:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993606:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572486.993608:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572486.993611:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572486.993614:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572486.993618:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72300 x1875440809959936/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572486.993627:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72300 x1875440809959936/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572502 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572486.993655:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572486.993659:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572486.993663:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572490.820100:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307629793, at 4307629826 00000100:00000010:0.0:1788572490.820111:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37025a45a0. 00000800:00000200:0.0:1788572491.013771:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788572491.013781:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572491.013793:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572491.013797:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572491.013816:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.111530:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111553:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.111557:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111560:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111570:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111579:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572492.111592:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111598:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111604:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111608:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417466400 00000400:00000200:0.0:1788572492.111614:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111621:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2e1 [1] + 224 00000800:00000001:0.0:1788572492.111629:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111647:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111650:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572492.111656:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572492.111664:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572492.111672:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72a00 x1875440809960448/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572492.111684:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72a00 x1875440809960448/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572492.111785:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572492.111798:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111802:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.111804:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111806:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111810:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111813:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572492.111817:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111820:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111822:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111824:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417466380 00000400:00000200:0.0:1788572492.111827:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111831:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2f1 [1] + 224 00000800:00000001:0.0:1788572492.111835:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111838:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111840:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572492.111843:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572492.111846:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572492.111851:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70380 x1875440809960320/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572492.111859:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec70380 x1875440809960320/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572492.111865:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572492.111868:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111871:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.111873:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111875:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111878:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111880:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572492.111884:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111886:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111888:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111890:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417466500 00000400:00000200:0.0:1788572492.111893:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111897:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2e9 [1] + 224 00000800:00000001:0.0:1788572492.111900:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111903:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111904:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572492.111907:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572492.111909:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572492.111913:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73800 x1875440809960704/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572492.111920:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73800 x1875440809960704/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572492.111927:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572492.111931:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111934:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.111935:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111937:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111940:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111944:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572492.111947:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111949:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111962:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111965:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417466300 00000400:00000200:0.0:1788572492.111968:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.111972:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2d9 [1] + 224 00000800:00000001:0.0:1788572492.111976:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.111980:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.111981:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572492.111985:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572492.111988:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572492.111992:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73480 x1875440809960192/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572492.111999:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73480 x1875440809960192/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572492.112005:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572492.112009:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.112012:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572492.112014:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.112016:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.112019:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.112021:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572492.112025:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.112027:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.112029:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.112031:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417466480 00000400:00000200:0.0:1788572492.112034:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572492.112037:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a2f9 [1] + 224 00000800:00000001:0.0:1788572492.112040:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.112048:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572492.112050:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572492.112053:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572492.112056:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572492.112060:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73100 x1875440809960576/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572492.112067:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73100 x1875440809960576/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572492.112073:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572492.112076:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572492.112078:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1788572494.195008:0:262629:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:0.0:1788572494.195013:0:262629:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity test 442: truncate vs read/write should not panic ========================================================== 21:41:32 (1788572492) 00000001:00000001:0.0:1788572494.212143:0:262629:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00000080:00000001:0.0:1788572494.459546:0:262641:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572494.459552:0:262641:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572494.459555:0:262641:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0e33de0 00000080:00200000:0.0:1788572494.459559:0:262641:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572494.459567:0:262641:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c371996cc00. 00800000:00000002:0.0:1788572494.459574:0:262641:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572494.459578:0:262641:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572494.459584:0:262641:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572494.459586:0:262641:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0e33d00 00800000:00000002:0.0:1788572494.459589:0:262641:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572494.459594:0:262641:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572494.459597:0:262641:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572494.459600:0:262641:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572494.459603:0:262641:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572494.459605:0:262641:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572494.459608:0:262641:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572494.459610:0:262641:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572494.459613:0:262641:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572494.459616:0:262641:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572494.459623:0:262641:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572494.459625:0:262641:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0e33bd0 00000002:00010000:0.0:1788572494.459627:0:262641:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572494.459632:0:262641:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572494.459654:0:262641:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572494.459658:0:262641:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572494.459666:0:262641:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37050f7f00 count: 2 00010000:00000001:0.0:1788572494.459670:0:262641:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572494.459676:0:262641:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572494.459682:0:262641:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572494.459687:0:262641:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c37421cb600/0x1441586093167cc0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a086d expref: -99 pid: 261255 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 02000000:00000001:0.0:1788572494.468781:0:262641:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 00000800:00000001:0.0:1788572494.481010:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.481028:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.481032:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.481035:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.481044:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.481052:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572494.481061:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.481067:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.481072:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.481076:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 328 into portal 15 MB=0x6a9b414921c00 00000400:00000200:0.0:1788572494.481085:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index f from 12345-192.168.202.148@tcp of length 328/328 into md 0x525bc1 [2] + 656 00000800:00000001:0.0:1788572494.481094:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.481109:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.481113:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.481118:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572494.481137:0:2396:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:0.0:1788572494.481139:0:2396:0:(events.c:300:request_in_callback()) event type 2, status 0, service ldlm_cbd 00000100:00000010:0.0:1788572494.481146:0:2396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7100. 00000100:00000040:0.0:1788572494.481150:0:2396:0:(events.c:341:request_in_callback()) incoming req@ffff9c372d6b7100 x1875440764591104 msgsize 328 00000100:00100000:0.0:1788572494.481155:0:2396:0:(events.c:344:request_in_callback()) peer: 12345-192.168.202.148@tcp (source: 12345-192.168.202.148@tcp) 00000100:00000001:0.0:1788572494.481171:0:2396:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:0.0:1788572494.481179:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.481183:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.511673:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.511679:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572494.684589:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.684593:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2900. 00000400:00000200:0.0:1788572494.684601:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.684606:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.684611:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3f68 00000400:00000010:0.0:1788572494.684614:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3f68. 00000100:00000001:0.0:1788572494.684617:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.684621:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7b80 x1875440809961344/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.684629:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.684632:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7b80 x1875440809961344/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.684667:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.684669:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572494.692841:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.692846:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d500. 00000400:00000200:0.0:1788572494.692851:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.692856:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.692860:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c14c8 00000400:00000010:0.0:1788572494.692862:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c14c8. 00000100:00000001:0.0:1788572494.692865:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.692869:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73480 x1875440809961472/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.692877:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.692879:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec73480 x1875440809961472/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.692886:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.692888:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572494.700805:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572494.700808:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572494.701053:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.701058:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2500. 00000400:00000200:0.0:1788572494.701065:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.701070:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.701075:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788572494.701078:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788572494.701081:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.701086:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.701095:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.701098:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.701106:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.701107:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572494.725266:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572494.725270:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572494.725277:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572494.725279:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572494.725283:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2b58. 00000400:00000200:0.0:1788572494.725287:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3088 00000400:00000010:0.0:1788572494.725290:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3088. 00000100:00000001:0.0:1788572494.725294:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572494.725298:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572494.725308:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.725314:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572494.725316:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572494.725319:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572494.725322:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572494.725325:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572494.725328:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572494.725331:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572494.725342:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce538 time=47 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572494.725347:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=90 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572494.725350:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572494.725352:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572494.725356:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572494.725360:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.725364:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572494.725367:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572494.725369:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572494.725371:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572494.725376:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.725384:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572494.725387:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572494.725389:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572494.725393:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572494.725402:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809961600:192.168.202.148@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1788572494.725407:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572494.725409:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.725413:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440809961600/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.725420:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572494.725422:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572494.725426:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3719b04000. 02000000:00000001:0.0:1788572494.725429:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572494.725431:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572494.725433:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572494.725435:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572494.725438:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3719b05000. 02000000:00000001:0.0:1788572494.725442:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572494.725444:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572494.725446:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788572494.725449:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572494.725451:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572494.725453:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572494.725461:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572494.725463:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.725465:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572494.725467:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572494.744285:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.744290:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2b00. 00000400:00000200:0.0:1788572494.744298:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.744303:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.744308:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572494.744311:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572494.744315:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.744321:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5c00 x1875440809961856/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.744331:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.744334:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5c00 x1875440809961856/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.744340:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.744342:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572494.754331:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.754351:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.754356:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.754358:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.754367:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.754375:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572494.754385:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.754390:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.754395:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.754399:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b417466980 00000400:00000200:0.0:1788572494.754404:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.754411:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a359 [1] + 224 00000800:00000001:0.0:1788572494.754418:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.754427:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.754431:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.754435:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572494.754443:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572494.754452:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5c00 x1875440809961856/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572494.754463:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371f1c5c00 x1875440809961856/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572579 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.754477:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000200:0.0:1788572494.754566:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.754570:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572494.754577:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.754580:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.754585:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1a18 00000400:00000010:0.0:1788572494.754587:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1a18. 00000100:00000001:0.0:1788572494.754591:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.754595:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69500 x1875440809962112/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572579 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.754606:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.754609:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed69500 x1875440809962112/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572579 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.754616:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.754618:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572494.754623:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.754626:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.768280:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.768296:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.768299:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.768301:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.768307:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768315:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572494.768324:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768329:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768334:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768338:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b417466b00 00000400:00000200:0.0:1788572494.768343:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768349:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a371 [1] + 224 00000800:00000001:0.0:1788572494.768356:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.768359:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.768362:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.768367:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572494.768373:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572494.768381:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73100 x1875440809962240/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572579 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572494.768392:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1152 req@ffff9c371ec73100 x1875440809962240/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572579 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.768453:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000200:0.0:1788572494.768536:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.768540:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d600. 00000400:00000200:0.0:1788572494.768545:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.768549:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.768553:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1aa0 00000400:00000010:0.0:1788572494.768555:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1aa0. 00000100:00000001:0.0:1788572494.768559:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.768563:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b800 x1875440809962368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.768574:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.768578:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440809962368/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.768586:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.768588:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572494.768593:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.768597:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.768599:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.768601:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.768604:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768608:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572494.768612:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768615:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768618:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768620:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b417466a80 00000400:00000200:0.0:1788572494.768624:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.768628:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a369 [32] + 224 00000800:00000001:0.0:1788572494.768632:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.770103:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.770109:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.770116:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572494.770123:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572494.770130:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69500 x1875440809962112/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572579 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572494.770140:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66320 req@ffff9c371ed69500 x1875440809962112/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572579 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572494.770161:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572494.770171:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.770174:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572494.773830:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572494.773835:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572494.773841:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.773845:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572494.773849:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1aa0 00000400:00000010:0.0:1788572494.773852:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1aa0. 00000100:00000001:0.0:1788572494.773856:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572494.773862:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809962496/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.773884:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572494.773888:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440809962496/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.773895:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572494.773897:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572494.779535:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.779552:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572494.779557:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.779560:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.779569:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.779578:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572494.779587:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.779592:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.779597:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.779602:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417466c00 00000400:00000200:0.0:1788572494.779606:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572494.779613:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a381 [1] + 224 00000800:00000001:0.0:1788572494.779620:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.779627:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572494.779631:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572494.779955:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572494.779964:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572494.779973:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68380 x1875440809962496/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572494.779985:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68380 x1875440809962496/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572510 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572494.780849:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572494.780862:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572494.780868:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.634400:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.634406:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d400. 00000400:00000200:0.0:1788572495.634412:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.634417:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.634423:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1880 00000400:00000010:0.0:1788572495.634426:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1880. 00000100:00000001:0.0:1788572495.634430:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.634435:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73100 x1875440809962752/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572580 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.634444:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.634447:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec73100 x1875440809962752/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572580 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.634454:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.634456:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.649401:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.649420:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.649424:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.649426:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.649435:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.649442:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.649450:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.649455:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.649460:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.649464:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b417466d00 00000400:00000200:0.0:1788572495.649469:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.649475:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a389 [32] + 224 00000800:00000001:0.0:1788572495.649482:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.649490:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.649493:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.649497:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.649504:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.649512:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73100 x1875440809962752/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572580 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.649523:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66320 req@ffff9c371ec73100 x1875440809962752/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 656/66320 e 0 to 0 dl 1788572580 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.649551:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.649560:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.649563:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.652699:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.652704:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6db00. 00000400:00000200:0.0:1788572495.652709:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.652715:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.652722:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1f68 00000400:00000010:0.0:1788572495.652725:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1f68. 00000100:00000001:0.0:1788572495.652729:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.652734:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73480 x1875440809963008/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572580 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.652743:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.652746:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec73480 x1875440809963008/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572580 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.652752:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.652753:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.658028:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.658045:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.658051:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.658053:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.658062:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.658070:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.658080:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.658085:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.658090:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.658094:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b417466e00 00000400:00000200:0.0:1788572495.658099:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.658105:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a391 [1] + 224 00000800:00000001:0.0:1788572495.658112:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.658119:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.658123:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.658138:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.658147:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.658156:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73480 x1875440809963008/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572580 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572495.658166:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=456 offset=224 replen=512 req@ffff9c371ec73480 x1875440809963008/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572580 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.658183:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.658192:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.658195:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.659496:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.659501:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d300. 00000400:00000200:0.0:1788572495.659506:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.659511:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.659515:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1770 00000400:00000010:0.0:1788572495.659517:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1770. 00000100:00000001:0.0:1788572495.659520:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.659524:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70380 x1875440809963136/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572580 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.659532:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.659535:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec70380 x1875440809963136/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572580 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.659541:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.659543:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.661857:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.661874:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.661885:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.661888:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.661896:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.661903:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.661911:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.661916:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.661922:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.661926:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 10 MB=0x6a9b417466e80 00000400:00000200:0.0:1788572495.661930:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.661936:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a399 [1] + 224 00000800:00000001:0.0:1788572495.661956:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.661963:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.661966:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.661970:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.661977:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.661985:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70380 x1875440809963136/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572580 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.661994:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=472 req@ffff9c371ec70380 x1875440809963136/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572580 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.662015:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.662023:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.662026:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.664510:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.664515:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572495.664519:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.664523:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.664527:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1ee0 00000400:00000010:0.0:1788572495.664529:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1ee0. 00000100:00000001:0.0:1788572495.664532:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.664535:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70380 x1875440809963264/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572511 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.664543:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.664545:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440809963264/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.664549:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.664551:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.667860:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.667884:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.667888:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.667891:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.667899:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.667906:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.667916:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.667921:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.667926:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.667929:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 432 into portal 4 MB=0x6a9b417466f00 00000400:00000200:0.0:1788572495.667934:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.667950:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 432/432 into md 0x52a3a1 [1] + 224 00000800:00000001:0.0:1788572495.667957:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.667964:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.667968:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.667972:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.667979:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.667987:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70380 x1875440809963264/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572511 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.667997:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=432 offset=224 replen=432 req@ffff9c371ec70380 x1875440809963264/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572511 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.668016:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.668024:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.668028:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.670743:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.670747:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d000. 00000400:00000200:0.0:1788572495.670750:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.670754:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.670758:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1198 00000400:00000010:0.0:1788572495.670760:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1198. 00000100:00000001:0.0:1788572495.670762:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.670765:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73800 x1875440809963520/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.670771:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.670774:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec73800 x1875440809963520/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.670778:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.670779:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.687211:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.687226:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.687230:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.687232:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.687238:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.687244:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.687250:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.687254:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.687258:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.687261:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b417467000 00000400:00000200:0.0:1788572495.687264:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.687269:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a3a9 [1] + 224 00000800:00000001:0.0:1788572495.687273:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.687278:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.687281:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.687283:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.687290:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.687297:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73800 x1875440809963520/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.687304:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=624 req@ffff9c371ec73800 x1875440809963520/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.687320:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.687326:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.687329:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.714707:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.714712:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572495.714717:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.714723:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.714728:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf14c8 00000400:00000010:0.0:1788572495.714731:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf14c8. 00000100:00000001:0.0:1788572495.714734:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.714739:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809963776/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 528/131856 e 0 to 0 dl 1788572580 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.714748:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.714751:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68380 x1875440809963776/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 528/131856 e 0 to 0 dl 1788572580 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.714758:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.714759:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.723602:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.723617:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.723621:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.723624:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.723633:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.723676:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.723686:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.723692:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.723697:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.723701:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b417467100 00000400:00000200:0.0:1788572495.723706:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.723711:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a3b1 [64] + 224 00000800:00000001:0.0:1788572495.723718:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.723726:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.723729:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.723733:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.723740:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.723748:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68380 x1875440809963776/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 528/131856 e 0 to 0 dl 1788572580 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.723757:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=131856 req@ffff9c371ed68380 x1875440809963776/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 528/131856 e 0 to 0 dl 1788572580 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.723772:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.723780:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.723783:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.728674:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.728679:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788572495.728685:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.728690:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.728697:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1f68 00000400:00000010:0.0:1788572495.728699:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1f68. 00000100:00000001:0.0:1788572495.728703:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.728708:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809963904/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.728718:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.728721:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6b480 x1875440809963904/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.728728:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.728729:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.748738:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.748755:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.748759:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.748762:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.748771:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.748779:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.748788:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.748792:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.748798:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.748815:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417467180 00000400:00000200:0.0:1788572495.748821:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.748827:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a3b9 [1] + 224 00000800:00000001:0.0:1788572495.748836:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.749071:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.749075:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.749080:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.749088:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.749098:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b480 x1875440809963904/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.749110:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ed6b480 x1875440809963904/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.749185:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.749193:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.749197:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.758693:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.758698:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6df00. 00000400:00000200:0.0:1788572495.758704:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.758710:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.758715:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1000 00000400:00000010:0.0:1788572495.758717:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1000. 00000100:00000001:0.0:1788572495.758721:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.758725:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72a00 x1875440809964032/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.758733:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.758736:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72a00 x1875440809964032/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.758741:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.758743:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.765710:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.765734:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.765738:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.765740:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.765749:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.765759:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.765768:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.765773:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.765778:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.765782:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417467200 00000400:00000200:0.0:1788572495.765787:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.765794:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a3c1 [1] + 224 00000800:00000001:0.0:1788572495.765816:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.765826:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.765830:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.765835:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.765853:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.765862:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72a00 x1875440809964032/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.765873:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec72a00 x1875440809964032/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572511 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.765906:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.765914:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.765917:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.777525:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.777534:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6da00. 00000400:00000200:0.0:1788572495.777542:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.777560:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.777567:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1088 00000400:00000010:0.0:1788572495.777570:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1088. 00000100:00000001:0.0:1788572495.777575:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.777590:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72a00 x1875440809964160/t0(0) o61->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 592/224 e 0 to 0 dl 1788572580 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.777603:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.777606:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec72a00 x1875440809964160/t0(0) o61->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 592/224 e 0 to 0 dl 1788572580 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.777612:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.777613:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.797322:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.797340:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.797345:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.797347:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.797355:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.797363:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.797371:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.797377:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.797382:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.797387:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467280 00000400:00000200:0.0:1788572495.797392:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.797398:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a3c9 [1] + 224 00000800:00000001:0.0:1788572495.797405:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.797411:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.797414:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.797419:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.797426:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.797435:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72a00 x1875440809964160/t0(0) o61->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 592/224 e 0 to 0 dl 1788572580 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572495.797446:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72a00 x1875440809964160/t0(0) o61->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 592/224 e 0 to 0 dl 1788572580 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.797471:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.797478:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.797481:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572495.800828:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.800834:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788572495.800838:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.800853:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.800858:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1770 00000400:00000010:0.0:1788572495.800861:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1770. 00000100:00000001:0.0:1788572495.800865:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.800869:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809964288/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.800887:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.800890:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440809964288/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.800896:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.800897:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572495.800909:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.800911:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6da00. 00000400:00000200:0.0:1788572495.800913:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.800916:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.800918:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c16e8 00000400:00000010:0.0:1788572495.800920:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c16e8. 00000100:00000001:0.0:1788572495.800922:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.800925:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70000 x1875440809964544/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.800930:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.800933:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec70000 x1875440809964544/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.800948:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.800950:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572495.801017:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572495.801020:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2300. 00000400:00000200:0.0:1788572495.801026:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.801029:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572495.801032:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3cc0 00000400:00000010:0.0:1788572495.801033:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3cc0. 00000100:00000001:0.0:1788572495.801036:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572495.801039:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6680 x1875440809964672/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.801046:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572495.801048:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6680 x1875440809964672/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.801053:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572495.801055:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572495.804252:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804268:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.804272:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804274:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.804282:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804289:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.804297:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804301:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804307:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804311:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417467300 00000400:00000200:0.0:1788572495.804316:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804322:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a3d1 [1] + 224 00000800:00000001:0.0:1788572495.804329:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804332:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.804335:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.804339:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.804346:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.804353:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b480 x1875440809964288/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572495.804364:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6b480 x1875440809964288/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.804425:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.804431:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804435:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.804437:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804439:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.804441:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804445:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.804449:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804451:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804454:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804457:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b417467480 00000400:00000200:0.0:1788572495.804460:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.804464:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a3e1 [1] + 224 00000800:00000001:0.0:1788572495.804468:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804473:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.804474:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.804478:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.804481:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.804485:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6680 x1875440809964672/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572495.804493:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c6680 x1875440809964672/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572511 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572495.804510:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.804513:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.804516:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.812289:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.812305:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572495.812309:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.812311:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.812319:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.812325:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572495.812334:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.812339:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.812344:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.812348:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b417467400 00000400:00000200:0.0:1788572495.812353:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572495.812359:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a3d9 [1] + 224 00000800:00000001:0.0:1788572495.812365:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.812427:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572495.812431:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572495.812436:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572495.812444:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572495.812451:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70000 x1875440809964544/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572495.812460:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=624 req@ffff9c371ec70000 x1875440809964544/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572511 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572495.812481:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572495.812489:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572495.812492:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572495.977775:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307634981, at 4307634984 00000100:00000010:0.0:1788572495.977791:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6a80. 00000800:00000001:0.0:1788572497.223518:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.223536:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.223539:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.223542:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.223727:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.223734:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572497.223744:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.223749:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.223753:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.223758:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417467580 00000400:00000200:0.0:1788572497.223763:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.223768:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a401 [1] + 224 00000800:00000001:0.0:1788572497.223774:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.223785:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.223788:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.223792:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572497.223798:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572497.223805:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b480 x1875440809964928/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572497.223816:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6b480 x1875440809964928/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.223825:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572497.223832:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.223836:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.228282:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.228301:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.228305:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.228307:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.228317:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.228325:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572497.228335:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.228340:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.228344:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.228348:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467600 00000400:00000200:0.0:1788572497.228353:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.228359:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a409 [1] + 224 00000800:00000001:0.0:1788572497.228366:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.228374:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.228377:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.228391:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572497.228398:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572497.228406:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68000 x1875440809965056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572497.228416:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68000 x1875440809965056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.228424:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000200:0.0:1788572497.228462:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572497.228466:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d300. 00000400:00000200:0.0:1788572497.228470:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.228474:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572497.228477:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1e58 00000400:00000010:0.0:1788572497.228480:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1e58. 00000100:00000001:0.0:1788572497.228483:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572497.228487:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68a80 x1875440809965184/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.228507:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572497.228511:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440809965184/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.228518:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572497.228520:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572497.228525:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.228529:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572497.228589:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572497.228592:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6db00. 00000400:00000200:0.0:1788572497.228595:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.228598:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572497.228601:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c12a8 00000400:00000010:0.0:1788572497.228603:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c12a8. 00000100:00000001:0.0:1788572497.228606:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572497.228610:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69c00 x1875440809965440/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.228617:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572497.228620:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440809965440/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.228626:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572497.228628:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572497.232051:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572497.232061:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d400. 00000400:00000200:0.0:1788572497.232069:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.232077:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572497.232085:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1bb0 00000400:00000010:0.0:1788572497.232087:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1bb0. 00000100:00000001:0.0:1788572497.232092:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572497.232099:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6bb80 x1875440809965312/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.232110:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572497.232113:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440809965312/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.232120:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572497.232122:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572497.244513:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244532:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.244536:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244540:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244561:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244570:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572497.244579:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244584:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244589:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244594:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417467780 00000400:00000200:0.0:1788572497.244599:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244605:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a419 [1] + 224 00000800:00000001:0.0:1788572497.244612:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244616:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244619:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.244623:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572497.244631:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572497.244693:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69c00 x1875440809965440/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572497.244705:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69c00 x1875440809965440/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.244725:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572497.244735:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244740:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.244741:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244743:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244746:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244749:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572497.244754:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244757:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244759:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244762:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467680 00000400:00000200:0.0:1788572497.244766:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244769:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a411 [1] + 224 00000800:00000001:0.0:1788572497.244773:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244777:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244779:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.244782:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572497.244785:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572497.244789:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68a80 x1875440809965184/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572497.244797:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68a80 x1875440809965184/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.244804:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572497.244808:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244811:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572497.244813:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244814:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244817:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244821:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572497.244825:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244828:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244830:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244833:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417467700 00000400:00000200:0.0:1788572497.244836:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572497.244839:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a421 [1] + 224 00000800:00000001:0.0:1788572497.244842:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244850:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572497.244852:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572497.244855:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572497.244857:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572497.244861:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6bb80 x1875440809965312/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572497.244868:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6bb80 x1875440809965312/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572497.244875:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572497.244879:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572497.244881:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572500.804520:0:237073:0:(service.c:1710:ptlrpc_at_check_timed()) Process entered 00000100:00000001:0.0:1788572500.804529:0:237073:0:(service.c:1721:ptlrpc_at_check_timed()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572501.057800:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307640031, at 4307640064 00000100:00000010:0.0:1788572501.057822:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37025a4840. 00000800:00000200:0.0:1788572502.349351:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572502.349367:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d100. 00000400:00000200:0.0:1788572502.349381:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.349402:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572502.349414:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1550 00000400:00000010:0.0:1788572502.349417:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1550. 00000100:00000001:0.0:1788572502.349429:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572502.349447:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70000 x1875440809965568/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349465:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572502.349469:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440809965568/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349476:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572502.349477:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572502.349496:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572502.349499:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572502.349503:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.349506:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572502.349508:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1c38 00000400:00000010:0.0:1788572502.349510:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1c38. 00000100:00000001:0.0:1788572502.349513:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572502.349518:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b100 x1875440809965696/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349526:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572502.349529:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440809965696/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349535:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572502.349537:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572502.349545:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572502.349547:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572502.349549:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.349553:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572502.349555:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788572502.349557:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788572502.349560:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572502.349564:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68700 x1875440809965952/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349577:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572502.349579:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440809965952/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349585:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572502.349586:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572502.349594:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572502.349596:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9900. 00000400:00000200:0.0:1788572502.349599:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.349601:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572502.349604:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1198 00000400:00000010:0.0:1788572502.349606:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1198. 00000100:00000001:0.0:1788572502.349608:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572502.349612:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809966080/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349618:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572502.349620:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809966080/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.349626:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572502.349628:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572502.350157:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572502.350161:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d500. 00000400:00000200:0.0:1788572502.350166:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.350170:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572502.350175:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1908 00000400:00000010:0.0:1788572502.350177:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1908. 00000100:00000001:0.0:1788572502.350180:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572502.350186:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70e00 x1875440809965824/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.350195:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572502.350198:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440809965824/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.350204:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572502.350205:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572502.354009:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354032:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.354038:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354041:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354053:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354062:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572502.354073:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354078:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354085:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354089:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467900 00000400:00000200:0.0:1788572502.354095:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354103:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a439 [1] + 224 00000800:00000001:0.0:1788572502.354124:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354129:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354134:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.354139:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572502.354150:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572502.354162:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809965824/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572502.354174:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec70e00 x1875440809965824/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.354198:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572502.354209:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354213:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.354215:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354216:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354219:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354223:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572502.354227:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354229:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354232:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354234:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417467800 00000400:00000200:0.0:1788572502.354238:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354242:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a429 [1] + 224 00000800:00000001:0.0:1788572502.354246:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354249:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354251:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.354254:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572502.354257:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572502.354263:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70000 x1875440809965568/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572502.354271:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec70000 x1875440809965568/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.354277:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572502.354280:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354283:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.354284:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354286:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354288:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354291:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572502.354295:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354296:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354299:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354302:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417467a00 00000400:00000200:0.0:1788572502.354305:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.354310:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a449 [1] + 224 00000800:00000001:0.0:1788572502.354314:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354323:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.354324:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.354327:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572502.354330:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572502.354335:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809966080/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572502.354344:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69880 x1875440809966080/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.354369:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572502.354373:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.354377:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.372247:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372285:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.372290:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372294:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.372307:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372316:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572502.372329:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372335:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372342:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372347:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467880 00000400:00000200:0.0:1788572502.372354:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372362:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a431 [1] + 224 00000800:00000001:0.0:1788572502.372371:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372375:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.372379:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.372384:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572502.372407:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572502.372419:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b100 x1875440809965696/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572502.372431:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6b100 x1875440809965696/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.372457:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572502.372472:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372479:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572502.372481:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372483:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.372486:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372489:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572502.372499:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372501:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372504:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372507:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417467980 00000400:00000200:0.0:1788572502.372511:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572502.372516:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a441 [1] + 224 00000800:00000001:0.0:1788572502.372520:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372528:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572502.372530:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572502.372534:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572502.372537:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572502.372542:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68700 x1875440809965952/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572502.372551:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68700 x1875440809965952/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572502.372558:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572502.372562:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572502.372565:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572505.924594:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572505.924601:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3718c6d600. 00000400:00000200:0.0:1788572505.924606:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572505.924615:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572505.924623:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1660 00000400:00000010:0.0:1788572505.924625:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1660. 00000100:00000001:0.0:1788572505.924630:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572505.924657:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70000 x1875440809966208/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572521 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.924669:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572505.924672:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec70000 x1875440809966208/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572521 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.924678:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572505.924680:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572505.937101:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.937126:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572505.937133:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.937135:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572505.937159:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.937169:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572505.937180:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.937186:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.937192:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.937197:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b417467a80 00000400:00000200:0.0:1788572505.937204:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.937212:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a451 [1] + 224 00000800:00000001:0.0:1788572505.937222:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.937229:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572505.937233:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572505.937238:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572505.937248:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572505.937260:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70000 x1875440809966208/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572521 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572505.937273:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371ec70000 x1875440809966208/t0(0) o101->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572521 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.937298:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572505.937310:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.937314:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572505.954863:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572505.954871:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37040c2a00. 00000400:00000200:0.0:1788572505.954883:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572505.954889:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572505.954907:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788572505.954910:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788572505.954915:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572505.954920:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6680 x1875440809966464/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572521 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.954930:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572505.954933:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6680 x1875440809966464/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572521 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.954939:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572505.954941:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572505.962323:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.962351:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572505.962358:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.962361:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572505.962374:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.962384:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572505.962395:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.962402:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.962409:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.962414:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b417467b80 00000400:00000200:0.0:1788572505.962419:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572505.962427:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a459 [1] + 224 00000800:00000001:0.0:1788572505.962437:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.962447:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572505.962452:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572505.962458:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572505.962471:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572505.962485:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6680 x1875440809966464/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572521 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572505.962498:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=624 req@ffff9c371f1c6680 x1875440809966464/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572521 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572505.962525:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572505.962537:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572505.962540:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572506.177973:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307645151, at 4307645184 00000100:00000010:0.0:1788572506.178000:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37025a4660. 00000800:00000200:0.0:1788572507.459676:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572507.459684:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9900. 00000400:00000200:0.0:1788572507.459692:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572507.459700:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572507.459706:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1990 00000400:00000010:0.0:1788572507.459709:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1990. 00000100:00000001:0.0:1788572507.459714:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572507.459720:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68700 x1875440809966720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459742:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572507.459747:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440809966720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572592 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459754:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572507.459756:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572507.459772:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572507.459774:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572507.459777:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572507.459780:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572507.459783:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1d48 00000400:00000010:0.0:1788572507.459784:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1d48. 00000100:00000001:0.0:1788572507.459786:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572507.459791:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69500 x1875440809966976/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459797:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572507.459800:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440809966976/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459806:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572507.459807:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572507.459816:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572507.459818:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572507.459821:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572507.459824:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572507.459827:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1880 00000400:00000010:0.0:1788572507.459829:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1880. 00000100:00000001:0.0:1788572507.459831:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572507.459834:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b100 x1875440809966592/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459841:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572507.459844:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440809966592/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459850:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572507.459852:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572507.459859:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572507.459861:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572507.459864:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572507.459867:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572507.459869:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1550 00000400:00000010:0.0:1788572507.459871:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1550. 00000100:00000001:0.0:1788572507.459873:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572507.459876:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809966848/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.459883:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572507.459886:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809966848/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572592 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.460247:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572507.460249:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572507.463846:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572507.463861:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572507.463876:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572507.463888:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572507.463909:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1220 00000400:00000010:0.0:1788572507.463913:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1220. 00000100:00000001:0.0:1788572507.463921:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572507.463933:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69c00 x1875440809967104/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.463950:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572507.463954:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440809967104/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572523 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572507.463962:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572507.463964:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572511.303229:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307650271, at 4307650309 00000100:00000010:0.0:1788572511.303248:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6a80. 00000800:00000001:0.0:1788572512.595915:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572512.595932:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572512.595937:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572512.595939:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572512.595950:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572512.595958:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572512.595967:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572512.595972:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572512.595977:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572512.595982:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417467f00 00000400:00000200:0.0:1788572512.595987:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572512.595993:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a4a9 [1] + 224 00000800:00000001:0.0:1788572512.596000:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572512.596008:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572512.596012:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572512.596017:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572512.596025:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572512.596034:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69c00 x1875440809967360/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572597 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572512.596045:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69c00 x1875440809967360/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572597 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572512.596056:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572512.596064:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572512.596077:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572513.991538:0:263024:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572514.909734:0:263086:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572516.172725:0:263180:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572516.418107:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307655391, at 4307655424 00000100:00000010:0.0:1788572516.418121:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6660. 00000020:00000001:0.0:1788572516.550811:0:252892:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572516.550814:0:252892:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572516.550818:0:252892:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572517.700427:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572517.700443:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572517.700458:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572517.700467:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572517.700537:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572517.700541:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572517.700612:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572517.700626:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0000. 00000100:00000001:0.0:1788572517.700685:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700705:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b4e00 x1875440809967872/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572517.700774:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572517.700781:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700790:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4e00 x1875440809967872/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572517.700841:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700845:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572517.700849:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572517.700863:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572517.700866:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572517.700869:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572517.700870:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572517.700872:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572517.700876:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0100. 00000100:00000001:0.0:1788572517.700880:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700884:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b6d80 x1875440809968000/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572517.700891:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572517.700894:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700897:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6d80 x1875440809968000/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572517.700905:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700907:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572517.700911:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572517.700914:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572517.700915:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572517.700918:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572517.700919:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572517.700921:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572517.700924:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0700. 00000100:00000001:0.0:1788572517.700926:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.700929:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID req@ffff9c372d6b5c00 x1875440809968128/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572517.701326:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572517.701332:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.701342:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5c00 x1875440809968128/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572517.701353:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.701355:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572517.701359:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1788572517.701412:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572517.701415:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572517.701417:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572517.701421:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572517.701423:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572517.701425:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572517.701426:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572517.701427:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572517.701433:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809968128/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572517.701439:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572517.701441:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572517.701443:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572517.701445:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572517.701447:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572517.701450:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809968128:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572517.701454:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572517.701457:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572517.701458:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572517.701460:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572517.701465:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3e400. 02000000:00000001:0.0:1788572517.701467:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572517.701859:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572517.701864:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8108. 00000400:00000010:0.0:1788572517.701870:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572517.701873:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809968128, portal 10 00000100:00000001:0.0:1788572517.701876:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572517.701877:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572517.701884:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b5c00 x1875440809968128/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572517.701891:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572517.701893:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572517.701896:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572517.701898:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809968128, offset 0 00000400:00000200:0.0:1788572517.701902:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572517.701907:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572517.701911:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572517.701914:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572517.701917:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572517.701919:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572517.701921:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572517.701923:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539936:539936:254:4294967295] 192.168.202.148@tcp LPNI seq info [539901:539901:6:4294967295] 00000400:00000200:0.0:1788572517.701927:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572517.701933:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572517.701938:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572517.701941:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788572517.701945:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572517.701951:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572517.701956:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0e00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572517.701962:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572517.701968:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26226 usecs (27 jiffies) 00000400:00000200:0.0:1788572517.701975:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572517.701979:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307656735 00000100:00000001:0.0:1788572517.701984:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307660513 00000100:00000001:0.0:1788572517.701989:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307656735 00000100:00000001:0.0:1788572517.701995:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307660513 00000100:00000001:0.0:1788572517.702000:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572517.702004:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572517.702008:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572517.702014:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572520.325237:0:263319:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572521.537712:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307660513, at 4307660544 00000100:00000010:0.0:1788572521.537728:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788572521.665854:0:252892:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572521.665858:0:252892:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572521.665861:0:252892:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572522.840362:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572522.840367:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572522.840374:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572522.840376:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572522.840380:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2b58. 00000400:00000200:0.0:1788572522.840384:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788572522.840386:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788572522.840389:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572522.840393:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440809968640/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572522.840401:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440809968640/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840407:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572522.840409:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.840411:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572522.840416:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572522.840418:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572522.840421:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572522.840424:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572522.840435:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=86 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572522.840439:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=118 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572522.840443:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572522.840444:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572522.840448:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572522.840453:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.840457:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572522.840460:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572522.840462:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572522.840464:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572522.840469:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440809968640/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840476:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572522.840479:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572522.840482:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572522.840485:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440809968640/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572522.840492:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809968640:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572522.840500:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572522.840501:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572522.840502:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2580. 00000400:00000200:0.0:1788572522.840506:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3330 00000400:00000010:0.0:1788572522.840507:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3330. 00000100:00000001:0.0:1788572522.840509:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572522.840513:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440809968896/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572522.840520:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440809968896/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840526:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572522.840527:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.840529:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572522.840531:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572522.840532:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572522.840533:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572522.840536:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572522.840540:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=86 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572522.840542:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=137 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572522.840544:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572522.840545:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572522.840547:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572522.840549:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.840550:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572522.840552:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572522.840553:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572522.840555:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572522.840558:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440809968896/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840564:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572522.840566:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572522.840568:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572522.840571:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440809968896/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572522.840578:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809968896:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572522.840584:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572522.840585:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572522.840586:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2dc0. 00000400:00000200:0.0:1788572522.840589:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f33b8 00000400:00000010:0.0:1788572522.840590:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f33b8. 00000100:00000001:0.0:1788572522.840592:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572522.840595:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440809968512/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572522.840603:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440809968512/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840609:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572522.840610:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.840612:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572522.840614:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572522.840615:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572522.840616:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572522.840618:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572522.840621:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=87 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572522.840623:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=87 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572522.840625:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572522.840626:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572522.840628:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572522.840629:0:2404:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.840631:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572522.840652:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440809968512/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840659:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572522.840661:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572522.840662:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572522.840665:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440809968512/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572522.840672:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:874b6a47-e66d-4b10-8108-059b044e1170:2404:1875440809968512:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572522.840678:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572522.840679:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572522.840680:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2d68. 00000400:00000200:0.0:1788572522.840683:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3d48 00000400:00000010:0.0:1788572522.840685:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3d48. 00000100:00000001:0.0:1788572522.840687:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572522.840689:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4700 x1875440809968768/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572522.840696:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4700 x1875440809968768/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840702:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572522.840703:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.840706:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572522.840707:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572522.840708:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572522.840709:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572522.840711:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572522.840715:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=86 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572522.840717:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=147 v=5 (1 1 0 0) 00000100:00000001:0.0:1788572522.840720:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572522.840721:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572522.840723:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572522.840724:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.840727:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572522.840728:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869947 00000100:00000001:0.0:1788572522.840730:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572522.840731:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572522.840734:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4700 x1875440809968768/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840740:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572522.840741:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572522.840743:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572522.840746:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4700 x1875440809968768/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572522.840753:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809968768:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572522.840758:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572522.840952:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572522.840955:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f21b8. 00000400:00000200:0.0:1788572522.840958:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f37f8 00000400:00000010:0.0:1788572522.840960:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f37f8. 00000100:00000001:0.0:1788572522.840963:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572522.840968:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440809969024/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572522.840976:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440809969024/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.840984:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572522.840985:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.840987:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572522.840989:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572522.840990:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572522.840992:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572522.840994:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572522.840999:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=86 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572522.841002:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=59 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572522.841004:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572522.841006:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572522.841008:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572522.841010:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.841012:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572522.841013:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010074 00000100:00000001:0.0:1788572522.841015:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572522.841017:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572522.841020:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440809969024/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841027:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572522.841029:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572522.841030:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572522.841033:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440809969024/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572522.841040:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809969024:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572522.841044:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841046:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572522.841050:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440809968640/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841057:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572522.841060:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572522.841063:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3f200. 02000000:00000001:0.0:1788572522.841068:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572522.841070:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572522.841073:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572522.841075:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572522.841078:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37040c2f00. 02000000:00000001:0.0:1788572522.841084:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572522.841086:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572522.841088:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788572522.841091:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572522.841093:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841095:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572522.841099:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440809968896/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841105:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572522.841106:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572522.841108:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3f600. 02000000:00000001:0.0:1788572522.841110:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572522.841111:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572522.841112:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572522.841114:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572522.841116:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37040c2300. 02000000:00000001:0.0:1788572522.841118:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572522.841119:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572522.841120:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788572522.841122:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572522.841123:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841125:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572522.841128:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440809968512/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841134:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572522.841135:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572522.841137:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3f400. 02000000:00000001:0.0:1788572522.841139:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572522.841140:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572522.841141:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572522.841144:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572522.841145:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37040c2100. 02000000:00000001:0.0:1788572522.841148:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572522.841149:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572522.841150:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788572522.841153:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572522.841154:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841155:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572522.841159:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4700 x1875440809968768/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572607 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841164:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572522.841166:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572522.841167:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3ea00. 02000000:00000001:0.0:1788572522.841169:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572522.841171:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572522.841172:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572522.841173:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572522.841174:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37040c2a00. 02000000:00000001:0.0:1788572522.841177:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572522.841177:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572522.841179:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4700. 00000100:00000001:0.0:1788572522.841181:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572522.841182:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841184:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572522.841187:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440809969024/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572538 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572522.841193:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572522.841194:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572522.841195:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3e200. 02000000:00000001:0.0:1788572522.841197:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572522.841198:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572522.841199:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572522.841200:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572522.841201:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37040c2b00. 02000000:00000001:0.0:1788572522.841203:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572522.841204:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572522.841205:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788572522.841207:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572522.841208:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841210:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572522.841218:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572522.841220:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572522.841222:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572522.841224:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572525.156444:0:263667:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572526.657951:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307665631, at 4307665664 00000100:00000010:0.0:1788572526.657967:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788572526.786052:0:252892:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572526.786056:0:252892:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572526.786059:0:252892:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1788572530.076923:0:263962:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572530.520986:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572530.520997:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572531.777823:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307670752, at 4307670784 00000100:00000010:0.0:1788572531.777842:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215f00. 00000020:00000001:0.0:1788572531.911999:0:252892:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572531.912003:0:252892:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572531.912006:0:252892:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572533.058723:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572533.058733:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572533.058742:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572533.058748:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572533.058752:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572533.058755:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572533.058760:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572533.058767:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0200. 00000100:00000001:0.0:1788572533.058774:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058785:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572533.058804:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572533.058809:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058815:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.058840:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058842:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572533.058845:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572533.058847:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572533.058849:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572533.058851:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572533.058852:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572533.058854:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572533.058856:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0d00. 00000100:00000001:0.0:1788572533.058859:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058862:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572533.058868:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572533.058869:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058872:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.058878:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058880:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572533.058883:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572533.058884:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572533.058886:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572533.058888:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572533.058889:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572533.058890:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572533.058893:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0100. 00000100:00000001:0.0:1788572533.058895:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058898:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572533.058909:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572533.058910:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058914:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.058920:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058922:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572533.058926:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572533.058929:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572533.058930:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572533.058932:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572533.058934:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572533.058935:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572533.058938:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0e00. 00000100:00000001:0.0:1788572533.058940:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058943:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572533.058950:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572533.058952:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058955:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.058962:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058964:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572533.058967:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572533.058970:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572533.058972:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 00000100:00000001:0.0:1788572533.058973:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572533.058975:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572533.058976:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572533.058979:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0b00. 00000100:00000001:0.0:1788572533.058982:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.058996:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572533.059003:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572533.059004:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059008:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059015:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059020:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13009) 00000100:00000001:0.0:1788572533.059032:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.059035:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572533.059037:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572533.059042:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059048:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572533.059050:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059053:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572533.059056:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572533.059058:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059061:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:874b6a47-e66d-4b10-8108-059b044e1170:2403:1875440809969920:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.059065:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572533.059071:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572533.059073:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059076:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572533.059079:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3ea00. 02000000:00000001:0.0:1788572533.059081:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059090:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572533.059095:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c60. 00000400:00000010:0.0:1788572533.059103:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572533.059112:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809969920, portal 25 00000100:00000001:0.0:1788572533.059114:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572533.059116:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000040:0.0:1788572533.059121:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059128:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572533.059130:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572533.059133:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572533.059136:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875440809969920, offset 0 00000400:00000200:0.0:1788572533.059141:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059151:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059160:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572533.059165:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572533.059169:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572533.059173:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572533.059175:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059179:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539950:539950:256:4294967295] 192.168.202.148@tcp LPNI seq info [539915:539915:8:4294967295] 00000400:00000200:0.0:1788572533.059184:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059192:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572533.059199:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572533.059203:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572533.059210:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572533.059219:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572533.059224:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572533.059237:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059241:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572533.059244:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307675871 00000100:00000010:0.0:1788572533.059249:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788572533.059256:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572533.059262:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307675871 00000100:00000010:0.0:1788572533.059265:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626240. 00000100:00000001:0.0:1788572533.059268:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572533.059271:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.059274:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059276:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572533.059279:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059285:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572533.059286:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059288:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572533.059289:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572533.059291:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059294:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970176:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.059297:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572533.059299:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572533.059300:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059302:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572533.059304:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3ec00. 02000000:00000001:0.0:1788572533.059305:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059308:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572533.059310:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8a50. 00000400:00000010:0.0:1788572533.059313:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572533.059316:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809970176, portal 10 00000100:00000001:0.0:1788572533.059317:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572533.059318:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572533.059323:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059328:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572533.059330:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572533.059332:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572533.059333:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809970176, offset 0 00000400:00000200:0.0:1788572533.059337:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059340:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059343:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572533.059345:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572533.059348:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572533.059349:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572533.059351:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059354:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539951:539951:255:4294967295] 192.168.202.148@tcp LPNI seq info [539916:539916:7:4294967295] 00000400:00000200:0.0:1788572533.059357:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059362:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572533.059366:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572533.059368:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572533.059372:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572533.059375:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572533.059377:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572533.059380:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059383:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26223 usecs (27 jiffies) 00000400:00000200:0.0:1788572533.059386:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572533.059387:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307672092 00000100:00000001:0.0:1788572533.059389:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059391:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307672092 00000100:00000001:0.0:1788572533.059393:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059395:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.059396:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059398:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572533.059400:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059405:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572533.059406:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059408:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572533.059409:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572533.059411:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059413:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970432:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.059417:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572533.059419:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572533.059420:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059422:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572533.059424:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3ee00. 02000000:00000001:0.0:1788572533.059426:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059428:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572533.059430:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c84d0. 00000400:00000010:0.0:1788572533.059433:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572533.059435:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809970432, portal 4 00000100:00000001:0.0:1788572533.059437:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572533.059438:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572533.059442:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059447:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572533.059448:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572533.059450:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572533.059452:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809970432, offset 0 00000400:00000200:0.0:1788572533.059455:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059458:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059460:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572533.059463:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572533.059464:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572533.059466:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572533.059468:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059470:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539952:539952:254:4294967295] 192.168.202.148@tcp LPNI seq info [539917:539917:6:4294967295] 00000400:00000200:0.0:1788572533.059473:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059478:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572533.059481:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572533.059484:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572533.059487:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572533.059490:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572533.059492:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572533.059494:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059496:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 171843 usecs (172 jiffies) 00000400:00000200:0.0:1788572533.059499:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572533.059500:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307672237 00000100:00000001:0.0:1788572533.059502:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059504:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307672237 00000100:00000001:0.0:1788572533.059506:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059507:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.059508:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059510:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059512:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.059518:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572533.059520:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572533.059522:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.059523:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572533.059524:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059526:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00000001:0.0:1788572533.059528:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.059529:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572533.059531:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.059532:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.059533:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572533.059534:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572533.059537:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059542:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572533.059544:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059545:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572533.059547:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572533.059548:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059551:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970048:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.059554:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572533.059556:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572533.059557:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059559:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572533.059562:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3f200. 02000000:00000001:0.0:1788572533.059563:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059566:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572533.059568:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788572533.059571:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572533.059586:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809970048, portal 10 00000100:00000001:0.0:1788572533.059588:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572533.059589:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572533.059595:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059600:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572533.059601:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572533.059604:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572533.059605:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809970048, offset 0 00000400:00000200:0.0:1788572533.059609:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059612:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059615:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572533.059617:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572533.059619:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572533.059621:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572533.059623:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059625:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539953:539953:253:4294967295] 192.168.202.148@tcp LPNI seq info [539918:539918:5:4294967295] 00000400:00000200:0.0:1788572533.059628:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059633:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572533.059672:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572533.059675:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572533.059678:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572533.059682:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572533.059684:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572533.059687:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059689:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15106 usecs (16 jiffies) 00000400:00000200:0.0:1788572533.059692:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572533.059693:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307672081 00000100:00000001:0.0:1788572533.059695:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059697:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307672081 00000100:00000001:0.0:1788572533.059698:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059700:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.059701:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059703:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572533.059706:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059711:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572533.059712:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059714:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572533.059715:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572533.059717:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572533.059719:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970304:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.059722:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572533.059724:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572533.059725:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.059727:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572533.059730:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c371ed3f000. 02000000:00000001:0.0:1788572533.059732:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572533.059734:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572533.059737:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8688. 00000400:00000010:0.0:1788572533.059739:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572533.059741:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809970304, portal 4 00000100:00000001:0.0:1788572533.059743:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572533.059744:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000040:0.0:1788572533.059749:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059753:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572533.059755:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572533.059757:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572533.059759:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809970304, offset 0 00000400:00000200:0.0:1788572533.059764:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059767:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059769:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572533.059771:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572533.059773:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572533.059776:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572533.059777:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572533.059780:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539954:539954:252:4294967295] 192.168.202.148@tcp LPNI seq info [539919:539919:4:4294967295] 00000400:00000200:0.0:1788572533.059783:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.059787:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572533.059800:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572533.059803:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572533.059806:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572533.059810:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572533.059812:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572533.059814:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.059817:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 163011 usecs (164 jiffies) 00000400:00000200:0.0:1788572533.059819:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572533.059821:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307672229 00000100:00000001:0.0:1788572533.059822:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059824:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307672229 00000100:00000001:0.0:1788572533.059826:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307675871 00000100:00000001:0.0:1788572533.059830:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.059831:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059833:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059836:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059849:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.059852:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572533.059896:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572533.059899:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788572533.059905:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.059909:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572533.059914:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31990 00000400:00000010:0.0:1788572533.059917:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31990. 00000100:00000001:0.0:1788572533.059921:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572533.059925:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059934:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.059937:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059945:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059947:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572533.059958:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572533.059960:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572533.059962:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.059966:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572533.059969:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788572533.059970:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788572533.059972:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572533.059975:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059981:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.059984:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.059990:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.059992:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572533.059999:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572533.060001:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0500. 00000400:00000200:0.0:1788572533.060003:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.060006:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572533.060008:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572533.060010:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572533.060012:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572533.060015:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060020:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.060023:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060029:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.060030:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572533.060036:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572533.060038:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788572533.060040:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.060043:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572533.060044:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788572533.060046:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788572533.060048:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572533.060051:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060058:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.060060:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060066:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.060067:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572533.060142:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572533.060145:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572533.060147:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.060151:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572533.060154:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572533.060155:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572533.060157:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572533.060160:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060168:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.060170:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.060176:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.060177:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572533.070425:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070447:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572533.070451:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070453:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070462:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070468:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572533.070477:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070483:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070488:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070492:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417468a00 00000400:00000200:0.0:1788572533.070497:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070502:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a589 [1] + 224 00000800:00000001:0.0:1788572533.070508:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070511:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070514:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.070518:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572533.070525:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070532:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572533.070542:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070564:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572533.070574:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070576:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572533.070578:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070579:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070582:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070585:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572533.070588:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070590:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070592:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070594:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417468980 00000400:00000200:0.0:1788572533.070597:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070600:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a5a9 [1] + 224 00000800:00000001:0.0:1788572533.070603:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070605:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070607:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.070609:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572533.070612:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070615:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572533.070622:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070628:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572533.070629:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070632:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572533.070633:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070660:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070663:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070666:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572533.070669:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070672:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070674:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070676:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417468900 00000400:00000200:0.0:1788572533.070679:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070682:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a579 [1] + 224 00000800:00000001:0.0:1788572533.070685:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070687:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070689:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.070691:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572533.070694:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070698:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572533.070706:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070713:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572533.070716:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070719:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572533.070721:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070735:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070739:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070742:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572533.070746:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070748:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070751:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070753:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417468b00 00000400:00000200:0.0:1788572533.070756:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070760:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a599 [1] + 224 00000800:00000001:0.0:1788572533.070763:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070766:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070768:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.070771:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572533.070774:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070778:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572533.070787:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070803:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572533.070808:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070811:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572533.070813:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070815:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070818:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070821:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572533.070825:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070826:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070829:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070831:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417468a80 00000400:00000200:0.0:1788572533.070834:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572533.070837:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a5b9 [1] + 224 00000800:00000001:0.0:1788572533.070841:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070848:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572533.070850:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572533.070853:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572533.070856:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070860:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572533.070867:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070874:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572533.070877:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572533.070880:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.070894:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.070897:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572533.070903:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572533.070905:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572533.070908:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788572533.070912:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31550 00000400:00000010:0.0:1788572533.070915:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31550. 00000100:00000001:0.0:1788572533.070918:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.070922:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572533.070929:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.070935:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572533.070937:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.070939:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572533.070942:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572533.070945:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572533.070947:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.070949:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572533.070960:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=97 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572533.070964:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=129 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572533.070966:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572533.070968:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572533.070970:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572533.070974:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.070978:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572533.070980:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572533.070982:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572533.070983:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.070999:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071006:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.071010:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572533.071012:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572533.071016:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572533.071029:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970048:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572533.071035:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572533.071036:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572533.071037:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8688. 00000400:00000200:0.0:1788572533.071040:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31bb0 00000400:00000010:0.0:1788572533.071042:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31bb0. 00000100:00000001:0.0:1788572533.071044:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.071047:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572533.071054:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071059:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572533.071061:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071062:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572533.071064:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572533.071065:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572533.071066:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.071067:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572533.071072:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=97 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572533.071075:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=148 v=5 (1 2 0 0) 00000100:00000001:0.0:1788572533.071077:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572533.071078:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572533.071080:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572533.071082:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.071084:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572533.071085:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572533.071087:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572533.071089:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.071092:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071098:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.071100:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572533.071101:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572533.071104:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572533.071110:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970304:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572533.071114:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572533.071115:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572533.071116:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c60. 00000400:00000200:0.0:1788572533.071119:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572533.071120:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572533.071122:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.071124:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572533.071131:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071135:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572533.071136:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071138:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572533.071139:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572533.071140:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572533.071141:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.071142:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572533.071145:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=98 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572533.071148:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=98 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572533.071150:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572533.071151:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572533.071152:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572533.071154:0:2403:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.071155:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.071158:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071164:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.071166:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572533.071167:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572533.071170:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572533.071176:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:874b6a47-e66d-4b10-8108-059b044e1170:2403:1875440809969920:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572533.071180:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572533.071182:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572533.071183:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8a50. 00000400:00000200:0.0:1788572533.071185:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572533.071186:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572533.071188:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.071190:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572533.071197:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071202:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572533.071203:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071204:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572533.071206:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572533.071207:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572533.071208:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.071210:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572533.071214:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=97 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572533.071216:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=8 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572533.071219:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572533.071220:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572533.071221:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572533.071223:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.071225:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572533.071226:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869947 00000100:00000001:0.0:1788572533.071227:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572533.071228:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.071232:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071238:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.071240:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572533.071241:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572533.071245:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572533.071252:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970176:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572533.071256:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572533.071257:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572533.071259:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c84d0. 00000400:00000200:0.0:1788572533.071261:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572533.071263:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572533.071264:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572533.071267:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572533.071274:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071281:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572533.071282:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071284:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572533.071285:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572533.071286:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572533.071287:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572533.071289:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572533.071293:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=97 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572533.071295:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=70 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572533.071297:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572533.071298:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572533.071299:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572533.071301:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.071303:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572533.071304:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010074 00000100:00000001:0.0:1788572533.071306:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572533.071308:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572533.071312:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071318:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572533.071319:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572533.071320:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572533.071323:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572533.071330:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970432:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572533.071333:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071336:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.071340:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4000 x1875440809970048/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071347:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572533.071349:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572533.071352:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3f200. 02000000:00000001:0.0:1788572533.071356:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572533.071359:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572533.071361:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572533.071363:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572533.071366:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0d00. 02000000:00000001:0.0:1788572533.071370:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572533.071372:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572533.071373:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4000. 00000100:00000001:0.0:1788572533.071376:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572533.071378:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071379:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.071382:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7480 x1875440809970304/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071387:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572533.071389:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572533.071391:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3f000. 02000000:00000001:0.0:1788572533.071393:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572533.071394:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572533.071395:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572533.071397:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572533.071398:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0e00. 02000000:00000001:0.0:1788572533.071400:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572533.071401:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572533.071402:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7480. 00000100:00000001:0.0:1788572533.071404:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572533.071405:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071406:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.071409:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6300 x1875440809969920/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071414:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572533.071415:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572533.071416:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3ea00. 02000000:00000001:0.0:1788572533.071418:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572533.071419:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572533.071420:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572533.071421:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572533.071422:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0200. 02000000:00000001:0.0:1788572533.071424:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572533.071425:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572533.071426:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6300. 00000100:00000001:0.0:1788572533.071428:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572533.071428:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071430:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.071432:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440809970176/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572618 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071436:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572533.071437:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572533.071438:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3ec00. 02000000:00000001:0.0:1788572533.071440:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572533.071441:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572533.071442:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572533.071444:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572533.071445:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0100. 02000000:00000001:0.0:1788572533.071446:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572533.071447:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572533.071448:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788572533.071450:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572533.071451:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071452:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572533.071456:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809970432/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572549 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572533.071461:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572533.071463:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572533.071464:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371ed3ee00. 02000000:00000001:0.0:1788572533.071466:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572533.071467:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572533.071468:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572533.071470:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572533.071471:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0b00. 02000000:00000001:0.0:1788572533.071473:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572533.071474:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572533.071475:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572533.071477:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572533.071479:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071481:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572533.071489:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572533.071490:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572533.071493:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572533.071496:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572534.798812:0:264234:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:0.0:1788572534.798818:0:264234:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity test_600b This test needs vmtouch utility 00000001:00000001:0.0:1788572534.801957:0:264234:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00000100:00000040:0.0:1788572536.904000:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307675871, at 4307675910 00000100:00000010:0.0:1788572536.904015:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788572538.186292:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572538.186308:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572538.188681:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.188696:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572538.188699:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.188701:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572538.188715:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.188721:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572538.188729:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.188733:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.188738:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.188741:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417468d00 00000400:00000200:0.0:1788572538.188746:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.188751:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a5d9 [1] + 224 00000800:00000001:0.0:1788572538.188758:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.188782:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572538.188785:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572538.188789:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572538.188797:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572538.188805:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5500 x1875440809970944/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572538.188815:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c5500 x1875440809970944/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.188833:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572538.188840:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.188843:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190218:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572538.190230:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572538.190234:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572538.190235:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572538.190238:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2c08. 00000400:00000200:0.0:1788572538.190241:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788572538.190244:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788572538.190246:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572538.190251:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440809970816/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572538.190259:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440809970816/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190266:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572538.190268:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190269:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572538.190271:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572538.190273:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572538.190275:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572538.190277:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572538.190282:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=102 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572538.190286:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=13 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572538.190288:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572538.190290:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572538.190292:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572538.190294:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.190306:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572538.190308:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869947 00000100:00000001:0.0:1788572538.190310:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572538.190312:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572538.190316:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440809970816/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190321:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572538.190324:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572538.190326:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572538.190330:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440809970816/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572538.190336:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970816:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572538.190342:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572538.190343:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572538.190344:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f26e0. 00000400:00000200:0.0:1788572538.190347:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3f68 00000400:00000010:0.0:1788572538.190349:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3f68. 00000100:00000001:0.0:1788572538.190351:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572538.190354:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440809971072/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572538.190360:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440809971072/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190365:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572538.190366:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190368:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572538.190369:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572538.190370:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572538.190371:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572538.190373:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572538.190377:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=102 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572538.190379:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=75 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572538.190381:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572538.190392:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572538.190394:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572538.190396:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.190398:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00100000:0.0:1788572538.190399:0:2403:0:(client.c:3133:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: committing for last_committed 4295010076 gen 7 00000100:00000040:0.0:1788572538.190404:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 4295010076) req@ffff9c371ed69c00 x1875440809969792/t4295010076(4295010076) o4->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:6/4 lens 496/448 e 0 to 0 dl 1788572546 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572538.190412:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572538.190415:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440809969792/t4295010076(4295010076) o4->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:6/4 lens 496/448 e 0 to 0 dl 1788572546 ref 1 fl Complete:RMQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572538.190422:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572538.190423:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572538.190426:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371ff2ec00. 02000000:00000001:0.0:1788572538.190429:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572538.190431:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572538.190433:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572538.190436:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572538.190439:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3703115400. 02000000:00000001:0.0:1788572538.190442:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572538.190443:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572538.190445:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788572538.190448:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572538.190449:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190450:0:2403:0:(client.c:3196:ptlrpc_free_committed()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788572538.190453:0:2403:0:(client.c:3243:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: normal: skip=1 replay=1/1 committed=0/0 00000100:00000001:0.0:1788572538.190455:0:2403:0:(client.c:3244:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572538.190456:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572538.190460:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440809971072/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190466:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572538.190468:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572538.190469:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572538.190473:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440809971072/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572538.190479:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809971072:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572538.190483:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572538.190484:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572538.190486:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2688. 00000400:00000200:0.0:1788572538.190488:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572538.190490:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572538.190492:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572538.190495:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440809970688/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572538.190502:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440809970688/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190508:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572538.190509:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190511:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572538.190512:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572538.190513:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572538.190515:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572538.190516:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572538.190520:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=102 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572538.190523:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=134 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572538.190525:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572538.190527:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572538.190528:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572538.190529:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.190531:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572538.190533:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572538.190534:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572538.190535:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572538.190539:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440809970688/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190545:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572538.190547:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572538.190549:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572538.190552:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440809970688/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572538.190559:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809970688:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572538.190563:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.190565:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572538.190568:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440809970816/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190574:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572538.190575:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572538.190577:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c371862f600. 02000000:00000001:0.0:1788572538.190581:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572538.190582:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572538.190583:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572538.190586:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572538.190587:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9900. 02000000:00000001:0.0:1788572538.190589:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572538.190590:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572538.190592:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788572538.190594:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572538.190595:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190597:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572538.190600:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440809971072/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190606:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572538.190607:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572538.190608:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37042e5a00. 02000000:00000001:0.0:1788572538.190611:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572538.190612:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572538.190613:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572538.190615:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572538.190616:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9700. 02000000:00000001:0.0:1788572538.190618:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572538.190619:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572538.190620:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788572538.190622:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572538.190623:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190625:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572538.190628:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440809970688/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572623 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.190633:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572538.190975:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572538.190977:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3703119200. 02000000:00000001:0.0:1788572538.190981:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572538.190982:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572538.190983:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572538.190985:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572538.190987:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9d00. 02000000:00000001:0.0:1788572538.190989:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572538.190990:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572538.190991:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788572538.190994:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572538.190995:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572538.190998:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.191000:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572538.191001:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572538.191003:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572538.191005:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572538.201190:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.201210:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572538.201215:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.201218:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572538.201238:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.201247:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572538.201256:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.201261:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.201267:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.201271:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417468b80 00000400:00000200:0.0:1788572538.201276:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572538.201281:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a5e9 [1] + 224 00000800:00000001:0.0:1788572538.201288:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.201464:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572538.201469:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572538.201474:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572538.201481:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572538.201490:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7b80 x1875440809970560/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572538.201501:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c7b80 x1875440809970560/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572538.201515:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572538.201523:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572538.201526:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572539.020978:0:264496:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572539.769432:0:264557:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572542.017879:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307680995, at 4307681024 00000100:00000010:0.0:1788572542.017885:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00000001:0.0:1788572543.302580:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572543.302589:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572543.302775:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306431:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572543.306440:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306445:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306454:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306461:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572543.306470:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306475:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306480:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306484:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417468e00 00000400:00000200:0.0:1788572543.306489:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306495:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a639 [1] + 224 00000800:00000001:0.0:1788572543.306502:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306505:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306509:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572543.306514:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572543.306521:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.306530:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572543.306542:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.306564:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572543.306573:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306577:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572543.306579:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306581:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306584:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306671:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572543.306678:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306680:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306683:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306764:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417468e80 00000400:00000200:0.0:1788572543.306769:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306774:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a619 [1] + 224 00000800:00000001:0.0:1788572543.306778:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306783:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306786:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572543.306789:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572543.306792:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.306797:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572543.306806:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.306813:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572543.306817:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306821:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572543.306822:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306824:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306828:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306832:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572543.306838:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306841:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306843:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306846:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417468f00 00000400:00000200:0.0:1788572543.306848:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572543.306851:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a649 [1] + 224 00000800:00000001:0.0:1788572543.306855:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306863:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572543.306865:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572543.306868:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572543.306871:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.306875:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572543.306882:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.306889:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572543.306892:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572543.306895:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.306914:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572543.306918:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572543.306923:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572543.306925:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572543.306928:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f24d0. 00000400:00000200:0.0:1788572543.306933:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3088 00000400:00000010:0.0:1788572543.306935:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3088. 00000100:00000001:0.0:1788572543.307385:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.307392:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572543.307401:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.307408:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572543.307410:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.307413:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572543.307416:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572543.307418:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572543.307420:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572543.307423:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572543.307431:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=108 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572543.307434:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=108 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572543.307437:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572543.307438:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572543.307441:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572543.307444:0:2404:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.307447:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572543.307452:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.307460:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572543.307463:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572543.307465:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572543.307469:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572543.307477:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:874b6a47-e66d-4b10-8108-059b044e1170:2404:1875440809971200:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572543.307483:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572543.307485:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572543.307486:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2aa8. 00000400:00000200:0.0:1788572543.307490:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3cc0 00000400:00000010:0.0:1788572543.307492:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3cc0. 00000100:00000001:0.0:1788572543.307494:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.307498:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572543.307506:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.307511:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572543.307513:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.307515:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572543.307516:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572543.307517:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572543.307519:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572543.307520:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572543.307527:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=107 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572543.307530:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=18 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572543.307532:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572543.307533:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572543.307535:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572543.307538:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.307541:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572543.307543:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869947 00000100:00000001:0.0:1788572543.307546:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572543.307547:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572543.307551:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.307558:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572543.307559:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572543.307561:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572543.307564:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572543.307572:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809971456:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572543.307577:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572543.307578:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572543.307580:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f20b0. 00000400:00000200:0.0:1788572543.307583:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f34c8 00000400:00000010:0.0:1788572543.307585:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f34c8. 00000100:00000001:0.0:1788572543.309089:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.309105:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572543.309117:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.309124:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572543.309127:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.309130:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572543.309133:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572543.309135:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572543.309137:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572543.309139:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572543.309145:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=107 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572543.309148:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=139 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572543.309150:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572543.309151:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572543.309154:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572543.309157:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.309160:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572543.309162:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572543.309164:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572543.309165:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572543.309169:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.309175:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572543.309177:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572543.309179:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572543.309183:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572543.309190:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809971328:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572543.309195:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.309198:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572543.309202:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7b80 x1875440809971200/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.309208:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572543.309210:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572543.309213:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37198a5800. 02000000:00000001:0.0:1788572543.309218:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572543.309220:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572543.309222:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572543.309224:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572543.309227:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c3720ff3900. 02000000:00000001:0.0:1788572543.309231:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572543.309232:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572543.309234:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7b80. 00000100:00000001:0.0:1788572543.309237:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572543.309238:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.309240:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572543.309245:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4000 x1875440809971456/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.309252:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572543.309253:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572543.309255:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708e33400. 02000000:00000001:0.0:1788572543.309259:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572543.309260:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572543.309261:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572543.309263:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572543.309264:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9100. 02000000:00000001:0.0:1788572543.309267:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572543.309268:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572543.309269:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4000. 00000100:00000001:0.0:1788572543.309272:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572543.309273:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.309275:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572543.309278:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440809971328/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572628 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.309284:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572543.309286:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572543.309287:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37198a5e00. 02000000:00000001:0.0:1788572543.309289:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572543.309290:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572543.309291:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572543.309293:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572543.309294:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9b00. 02000000:00000001:0.0:1788572543.309297:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572543.309298:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572543.309299:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788572543.309301:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572543.309302:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.309306:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.309308:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572543.310239:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572543.310245:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572543.310247:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572543.310249:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2c60. 00000400:00000200:0.0:1788572543.310254:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572543.310256:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572543.310259:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.310264:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440809971584/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572543.310272:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440809971584/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310279:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572543.310280:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.310282:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572543.310284:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572543.310286:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572543.310288:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572543.310290:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572543.310298:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=107 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572543.310301:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=8 v=5 (1 1 2 0) 00000100:00000001:0.0:1788572543.310303:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572543.310304:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572543.310306:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572543.310309:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.310668:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572543.310670:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572543.310672:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572543.310673:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572543.310679:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440809971584/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310685:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572543.310766:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572543.310767:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572543.310773:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440809971584/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572543.310781:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809971584:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572543.310788:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.310790:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572543.310793:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440809971584/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310800:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572543.310801:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572543.310803:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c37198a5200. 02000000:00000001:0.0:1788572543.310807:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572543.310809:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572543.310811:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572543.310813:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572543.310815:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9d00. 02000000:00000001:0.0:1788572543.310818:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572543.310820:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572543.310822:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788572543.310826:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572543.310828:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.310831:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.310832:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572543.310834:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572543.310837:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572543.310838:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572543.310840:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2630. 00000400:00000200:0.0:1788572543.310842:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788572543.310844:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788572543.310846:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572543.310850:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440809971712/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572543.310858:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440809971712/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310865:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572543.310866:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.310868:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572543.310870:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572543.310871:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572543.310873:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572543.310875:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572543.310880:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=107 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572543.310883:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=80 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572543.310885:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572543.310886:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572543.310888:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572543.310890:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.310892:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572543.310893:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010076 00000100:00000001:0.0:1788572543.310895:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572543.310896:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572543.310900:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440809971712/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310907:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572543.310908:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572543.310910:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572543.310913:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440809971712/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572543.310919:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809971712:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572543.310923:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.310924:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572543.310927:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440809971712/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572559 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572543.310932:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572543.310933:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572543.310935:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708e33c00. 02000000:00000001:0.0:1788572543.311386:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572543.311389:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572543.311391:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572543.311394:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572543.311396:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9700. 02000000:00000001:0.0:1788572543.311399:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572543.311400:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572543.311402:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788572543.311405:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572543.311406:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.311408:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572543.311416:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572543.311418:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572543.311421:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572543.311423:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572543.635256:0:264794:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572547.139079:0:216275:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307686110, at 4307686145 00000100:00000010:0.0:1788572547.139091:0:216275:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788572548.290556:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572548.290560:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572548.290566:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572548.424142:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572548.424147:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572548.424155:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.424161:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572548.424166:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1550 00000400:00000010:0.0:1788572548.424168:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1550. 00000100:00000001:0.0:1788572548.424173:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572548.424178:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68700 x1875440809971840/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424187:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572548.424190:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440809971840/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572564 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424197:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572548.424199:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572548.424209:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572548.424211:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5e00. 00000400:00000200:0.0:1788572548.424214:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.424216:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572548.424219:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1990 00000400:00000010:0.0:1788572548.424220:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1990. 00000100:00000001:0.0:1788572548.424223:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572548.424226:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69500 x1875440809972096/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424232:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572548.424235:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440809972096/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424241:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572548.424243:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572548.424299:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572548.424302:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5600. 00000400:00000200:0.0:1788572548.424305:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.424308:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572548.424310:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788572548.424311:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788572548.424314:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572548.424317:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68a80 x1875440809972352/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424324:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572548.424327:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440809972352/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.424333:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572548.424334:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572548.427526:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437149:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.437160:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437165:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437174:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437183:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572548.437192:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437197:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437202:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437207:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469180 00000400:00000200:0.0:1788572548.437212:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437218:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a671 [1] + 224 00000800:00000001:0.0:1788572548.437225:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437228:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437232:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.437236:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572548.437244:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572548.437254:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69500 x1875440809972096/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572548.437264:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69500 x1875440809972096/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.437277:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572548.437284:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437288:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.437289:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437292:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437294:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437298:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572548.437301:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437303:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437306:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437308:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417469080 00000400:00000200:0.0:1788572548.437311:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437316:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a669 [1] + 224 00000800:00000001:0.0:1788572548.437320:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437323:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437325:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.437328:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572548.437330:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572548.437335:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68700 x1875440809971840/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572548.437349:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68700 x1875440809971840/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.437356:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572548.437360:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437363:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.437365:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437367:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437370:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437374:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572548.437377:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437380:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437382:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437384:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469280 00000400:00000200:0.0:1788572548.437388:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.437391:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a679 [1] + 224 00000800:00000001:0.0:1788572548.437394:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437401:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.437403:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.437406:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572548.437409:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572548.437413:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68a80 x1875440809972352/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572548.437422:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68a80 x1875440809972352/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.437428:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572548.437431:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.437434:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.444728:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444754:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.444769:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444773:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.444782:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444789:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572548.444798:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444802:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444807:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444810:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469200 00000400:00000200:0.0:1788572548.444815:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444821:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a689 [1] + 224 00000800:00000001:0.0:1788572548.444827:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444831:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.444834:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.444838:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572548.444845:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572548.444853:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809972224/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572548.444863:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69880 x1875440809972224/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572564 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.444879:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572548.444888:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444891:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572548.444893:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444895:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.444898:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444901:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572548.444905:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444907:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444910:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444912:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469100 00000400:00000200:0.0:1788572548.444915:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572548.444919:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a681 [1] + 224 00000800:00000001:0.0:1788572548.444922:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.444976:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572548.444979:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572548.444983:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572548.444986:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572548.444990:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b100 x1875440809971968/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572548.444998:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6b100 x1875440809971968/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572633 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572548.445005:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572548.445009:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572548.445013:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572552.258539:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307691234, at 4307691264 00000100:00000010:0.0:1788572552.258555:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6a80. 00000020:00000001:0.0:1788572553.415449:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572553.415456:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572553.415461:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572553.538837:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572553.538848:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572553.539104:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572553.539112:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572553.539117:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572553.539120:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572553.539126:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572553.539135:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0e00. 00000100:00000001:0.0:1788572553.539144:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572553.539156:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572553.539167:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572553.539171:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572553.539177:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.540023:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572553.540028:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572553.540031:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1788572553.540093:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572553.540096:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572553.544718:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572553.544722:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5e00. 00000400:00000200:0.0:1788572553.544726:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.544732:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572553.544736:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1aa0 00000400:00000010:0.0:1788572553.544738:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1aa0. 00000100:00000001:0.0:1788572553.544742:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572553.544746:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73100 x1875440809972608/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.544754:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572553.544757:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73100 x1875440809972608/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.544765:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.544767:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572553.549532:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572553.549535:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572553.549542:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572553.549544:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572553.549549:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2e70. 00000400:00000200:0.0:1788572553.549553:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572553.549556:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572553.549558:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572553.549563:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572553.549570:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.549576:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572553.549591:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572553.549593:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572553.549596:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572553.549598:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572553.549601:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572553.549604:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572553.549611:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=118 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572553.549615:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=118 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572553.549617:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572553.549619:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572553.549622:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572553.549625:0:2404:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.549628:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572553.549632:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.549875:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572553.549880:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572553.549882:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572553.549888:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572553.549896:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:874b6a47-e66d-4b10-8108-059b044e1170:2404:1875440809972480:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572553.549900:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572553.549903:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572553.549906:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809972480/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.549913:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572553.549916:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572553.549918:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3730310e00. 02000000:00000001:0.0:1788572553.549922:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572553.549924:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572553.549926:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572553.549928:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572553.549932:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0e00. 02000000:00000001:0.0:1788572553.549941:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572553.549943:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572553.549945:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572553.549949:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572553.549951:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572553.549953:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572553.549957:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572553.549959:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.549961:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572553.549963:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572553.552012:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572553.552016:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572553.552023:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.552028:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572553.552033:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1e58 00000400:00000010:0.0:1788572553.552035:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1e58. 00000100:00000001:0.0:1788572553.552039:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572553.552043:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809972736/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552053:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572553.552056:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440809972736/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552062:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.552063:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572553.552074:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572553.552076:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5a00. 00000400:00000200:0.0:1788572553.552079:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.552082:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572553.552085:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1cc0 00000400:00000010:0.0:1788572553.552086:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1cc0. 00000100:00000001:0.0:1788572553.552088:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572553.552092:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809972864/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552099:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572553.552109:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809972864/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552114:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.552116:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572553.552177:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572553.552180:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498b00. 00000400:00000200:0.0:1788572553.552183:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.552187:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572553.552190:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c17f8 00000400:00000010:0.0:1788572553.552192:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c17f8. 00000100:00000001:0.0:1788572553.552195:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572553.552198:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809972992/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552206:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572553.552208:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440809972992/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.552215:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572553.552217:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572553.572428:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572445:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572553.572450:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572452:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572462:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572469:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572553.572478:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572482:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572487:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572491:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469400 00000400:00000200:0.0:1788572553.572495:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572501:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6a9 [1] + 224 00000800:00000001:0.0:1788572553.572508:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572512:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572515:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.572533:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572553.572542:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572553.572552:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809972736/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572553.572562:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72680 x1875440809972736/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572638 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.572582:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572553.572592:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572596:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572553.572597:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572763:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572768:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572772:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572553.572778:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572780:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572784:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572787:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469480 00000400:00000200:0.0:1788572553.572791:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572795:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6b9 [1] + 224 00000800:00000001:0.0:1788572553.572799:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572803:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572805:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.572808:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572553.572811:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572553.572816:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809972864/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572553.572824:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec71f80 x1875440809972864/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.572831:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572553.572835:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572846:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572553.572848:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572850:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572854:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572857:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572553.572962:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572966:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572969:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572972:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469500 00000400:00000200:0.0:1788572553.572975:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572553.572979:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6b1 [1] + 224 00000800:00000001:0.0:1788572553.572983:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.572990:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572553.572992:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572553.572995:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572553.572998:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572553.573003:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809972992/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572553.573010:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec71500 x1875440809972992/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572553.573029:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572553.573034:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572553.573037:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572557.379902:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307696351, at 4307696386 00000100:00000010:0.0:1788572557.379919:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788572558.529908:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572558.529911:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572558.529914:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572558.657922:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572558.657933:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572558.657945:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572558.657951:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572558.657980:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572558.657983:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572558.657989:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572558.657998:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0e00. 00000100:00000001:0.0:1788572558.658004:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658017:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572558.658026:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572558.658031:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658038:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658075:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658078:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572558.658081:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572558.658085:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572558.658087:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572558.658089:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572558.658090:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572558.658092:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572558.658095:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0d00. 00000100:00000001:0.0:1788572558.658098:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658103:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572558.658110:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572558.658112:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658116:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658123:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658125:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572558.658128:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572558.658137:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572558.658139:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572558.658142:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572558.658143:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572558.658146:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572558.658149:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0200. 00000100:00000001:0.0:1788572558.658152:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658157:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572558.658164:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572558.658166:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658169:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658176:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658178:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572558.658181:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572558.658184:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572558.658186:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572558.658188:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572558.658189:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572558.658190:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572558.658193:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0400. 00000100:00000001:0.0:1788572558.658196:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658200:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572558.658206:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572558.658207:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658210:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658217:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658224:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572558.658226:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572558.658229:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572558.658232:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000100:00000001:0.0:1788572558.658233:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572558.658235:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572558.658236:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572558.658239:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0800. 00000100:00000001:0.0:1788572558.658242:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658245:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572558.658252:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572558.658253:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658256:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658264:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658267:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13035) 00000100:00000001:0.0:1788572558.658280:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572558.658281:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572558.658282:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572558.658287:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658293:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572558.658294:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.658297:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572558.658298:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572558.658300:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572558.658303:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809973248:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572558.658306:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572558.658308:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572558.658309:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.658311:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572558.658318:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3708d35a00. 02000000:00000001:0.0:1788572558.658319:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572558.658322:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572558.658325:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c84d0. 00000400:00000010:0.0:1788572558.658329:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572558.658331:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809973248, portal 10 00000100:00000001:0.0:1788572558.658333:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572558.658334:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572558.658340:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658346:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572558.658348:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572558.658350:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572558.658352:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809973248, offset 0 00000400:00000200:0.0:1788572558.658356:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572558.658673:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.658678:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572558.658680:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572558.658683:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572558.658686:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572558.658688:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572558.658691:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539976:539976:255:4294967295] 192.168.202.148@tcp LPNI seq info [539941:539941:7:4294967295] 00000400:00000200:0.0:1788572558.658695:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.658702:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572558.658707:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572558.658710:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572558.658713:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572558.658718:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572558.658721:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572558.658724:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658728:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15110 usecs (16 jiffies) 00000400:00000200:0.0:1788572558.658736:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572558.658738:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307697681 00000100:00000001:0.0:1788572558.658740:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.658742:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307697681 00000100:00000001:0.0:1788572558.658745:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.658747:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572558.658749:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.658751:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572558.658755:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658762:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572558.658763:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.658765:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572558.658767:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572558.658769:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572558.658772:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809973504:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572558.658775:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572558.658777:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572558.658779:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.658836:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572558.658839:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3708d34400. 02000000:00000001:0.0:1788572558.658842:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572558.658846:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572558.658848:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8a50. 00000400:00000010:0.0:1788572558.658852:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572558.658855:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809973504, portal 4 00000100:00000001:0.0:1788572558.658857:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572558.658859:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000040:0.0:1788572558.658866:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.658873:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572558.658880:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572558.658882:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572558.658885:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809973504, offset 0 00000400:00000200:0.0:1788572558.658889:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572558.658893:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.658897:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572558.658899:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572558.658902:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572558.658904:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572558.658906:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572558.658909:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539977:539977:254:4294967295] 192.168.202.148@tcp LPNI seq info [539942:539942:6:4294967295] 00000400:00000200:0.0:1788572558.658912:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.658918:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572558.658922:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572558.658925:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572558.658929:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572558.658933:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572558.658936:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572558.658939:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.658943:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 162283 usecs (163 jiffies) 00000400:00000200:0.0:1788572558.658946:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572558.658948:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307697828 00000100:00000001:0.0:1788572558.658950:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.658952:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307697828 00000100:00000001:0.0:1788572558.658954:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.658956:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572558.658958:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.658961:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.659021:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572558.659029:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572558.659036:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572558.659039:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572558.659040:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572558.659042:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572558.659045:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572558.659046:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572558.659049:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572558.659051:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572558.659052:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572558.659053:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572558.659055:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572558.659059:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659066:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572558.659067:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.659070:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572558.659071:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572558.659074:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572558.659076:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809973632:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572558.659080:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572558.659082:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572558.659084:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572558.659086:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572558.659089:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3708d35800. 02000000:00000001:0.0:1788572558.659091:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572558.659095:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572558.659097:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c60. 00000400:00000010:0.0:1788572558.659100:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572558.659103:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809973632, portal 4 00000100:00000001:0.0:1788572558.659105:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572558.659107:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572558.659112:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659119:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572558.659125:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572558.659127:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572558.659130:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809973632, offset 0 00000400:00000200:0.0:1788572558.659134:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572558.659138:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.659142:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572558.659144:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572558.659147:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572558.659149:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572558.659151:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572558.659154:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [539978:539978:253:4294967295] 192.168.202.148@tcp LPNI seq info [539943:539943:5:4294967295] 00000400:00000200:0.0:1788572558.659158:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.659164:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572558.659168:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572558.659171:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572558.659175:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572558.659179:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572558.659182:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572558.659185:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572558.659188:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 171033 usecs (172 jiffies) 00000400:00000200:0.0:1788572558.659191:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572558.659192:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307697837 00000100:00000001:0.0:1788572558.659195:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.659197:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307697837 00000100:00000001:0.0:1788572558.659199:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307701470 00000100:00000001:0.0:1788572558.659201:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572558.659204:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.659206:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.659209:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572558.659268:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572558.659277:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572558.659284:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.659289:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572558.659294:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f35d8 00000400:00000010:0.0:1788572558.659296:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f35d8. 00000100:00000001:0.0:1788572558.659301:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572558.659305:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659313:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572558.659316:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659322:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.659324:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572558.659335:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572558.659337:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0500. 00000400:00000200:0.0:1788572558.659339:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.659342:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572558.659344:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572558.659346:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572558.659348:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572558.659351:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659357:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572558.659360:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.659378:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.659379:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572558.659388:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572558.659390:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572558.659392:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.660294:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572558.660299:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31bb0 00000400:00000010:0.0:1788572558.660302:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31bb0. 00000100:00000001:0.0:1788572558.660314:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572558.660320:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.660328:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572558.660331:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.660337:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.660339:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572558.662703:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572558.663121:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788572558.663128:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.663136:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572558.663143:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572558.663146:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572558.663151:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572558.663159:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.663170:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572558.663173:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.663180:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.663182:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572558.663211:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572558.663214:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572558.663218:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.663221:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572558.663224:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3a18 00000400:00000010:0.0:1788572558.663226:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3a18. 00000100:00000001:0.0:1788572558.663229:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572558.663233:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.663241:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572558.663257:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.663263:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572558.663265:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572558.669777:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.669793:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.669798:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.669800:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.669809:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.669817:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572558.669826:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.669831:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.669836:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.669840:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417469580 00000400:00000200:0.0:1788572558.669845:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.669850:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6c1 [1] + 224 00000800:00000001:0.0:1788572558.669857:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.669863:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.669867:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.669871:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572558.669878:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572558.669888:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572558.669898:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6680 x1875440809973120/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.669920:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572558.669927:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.669930:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.681281:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681302:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.681307:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681310:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.681320:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681344:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572558.681354:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681359:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681364:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681368:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469600 00000400:00000200:0.0:1788572558.681373:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681379:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6d1 [1] + 224 00000800:00000001:0.0:1788572558.681388:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681394:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.681399:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.681420:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572558.681430:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572558.681441:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572558.681454:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4700 x1875440809973248/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.681491:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572558.681505:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681511:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.681513:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681515:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.681519:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681524:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572558.681529:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681531:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681535:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681538:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469780 00000400:00000200:0.0:1788572558.681542:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.681548:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a701 [1] + 224 00000800:00000001:0.0:1788572558.681552:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.681608:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.681618:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.681622:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572558.681627:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572558.681633:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572558.682576:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b4000 x1875440809973632/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.682585:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572558.682593:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.682598:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.684993:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.685010:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.685015:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.685017:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.685025:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.685032:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572558.685041:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.685045:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.685050:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.685055:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469680 00000400:00000200:0.0:1788572558.685060:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.685066:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6e1 [1] + 224 00000800:00000001:0.0:1788572558.685072:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.685082:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.685086:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.685091:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572558.685097:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572558.685105:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572558.685117:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b6300 x1875440809973376/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572643 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.685157:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572558.685165:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.685169:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.696702:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.696718:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572558.696722:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.696725:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.696734:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.696742:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572558.696751:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.696755:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.696760:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.696764:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469700 00000400:00000200:0.0:1788572558.696769:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572558.696776:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a6f1 [1] + 224 00000800:00000001:0.0:1788572558.696782:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.696836:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572558.696839:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572558.696844:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572558.696851:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572558.696860:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572558.696870:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7480 x1875440809973504/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572558.696890:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572558.696897:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572558.696901:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572562.497766:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307701470, at 4307701504 00000100:00000010:0.0:1788572562.497784:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ea0. 00000020:00000001:0.0:1788572563.651988:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572563.651997:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572563.652000:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572563.781518:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572563.781522:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498000. 00000400:00000200:0.0:1788572563.781529:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.781534:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572563.781539:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1220 00000400:00000010:0.0:1788572563.781542:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1220. 00000100:00000001:0.0:1788572563.781546:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572563.781551:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70a80 x1875440809973888/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.781560:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572563.781563:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440809973888/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.781571:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572563.781573:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572563.781648:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572563.781651:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5a00. 00000400:00000200:0.0:1788572563.781654:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.781657:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572563.781660:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf17f8 00000400:00000010:0.0:1788572563.781662:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf17f8. 00000100:00000001:0.0:1788572563.781664:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572563.781668:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71500 x1875440809973760/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.781675:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572563.781678:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440809973760/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.781684:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572563.781686:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572563.782756:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572563.782760:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498d00. 00000400:00000200:0.0:1788572563.782764:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.782774:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572563.782778:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1cc0 00000400:00000010:0.0:1788572563.782780:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1cc0. 00000100:00000001:0.0:1788572563.782783:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572563.782787:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809974144/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.782796:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572563.782799:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809974144/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.782804:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572563.782806:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572563.794672:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794683:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.794686:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794689:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.794695:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794700:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572563.794706:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794710:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794714:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794718:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b417469800 00000400:00000200:0.0:1788572563.794722:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794727:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a719 [1] + 224 00000800:00000001:0.0:1788572563.794733:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794736:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.794739:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.794744:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572563.794750:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572563.794756:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71500 x1875440809973760/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572563.794765:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec71500 x1875440809973760/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.794791:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000200:0.0:1788572563.794821:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572563.794833:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5400. 00000400:00000200:0.0:1788572563.794836:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.794840:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572563.794843:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf15d8 00000400:00000010:0.0:1788572563.794845:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf15d8. 00000100:00000001:0.0:1788572563.794849:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572563.794852:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec72680 x1875440809974016/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.794860:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572563.794863:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440809974016/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.794870:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572563.794872:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572563.794876:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794880:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.794882:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794884:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.794887:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794891:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572563.794894:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794897:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794899:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794902:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469880 00000400:00000200:0.0:1788572563.794905:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.794908:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a711 [1] + 224 00000800:00000001:0.0:1788572563.794912:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.794917:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.794919:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.794922:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572563.794932:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572563.794937:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70a80 x1875440809973888/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572563.794945:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec70a80 x1875440809973888/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.794954:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000200:0.0:1788572563.794996:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572563.794999:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5e00. 00000400:00000200:0.0:1788572563.795002:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.795005:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572563.795008:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1330 00000400:00000010:0.0:1788572563.795010:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1330. 00000100:00000001:0.0:1788572563.795012:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572563.795015:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73100 x1875440809974272/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.795021:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572563.795024:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73100 x1875440809974272/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.795030:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572563.795031:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572563.795035:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.795039:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.801031:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801048:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.801053:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801055:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.801063:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801071:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572563.801080:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801084:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801089:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801093:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469900 00000400:00000200:0.0:1788572563.801112:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801118:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a729 [1] + 224 00000800:00000001:0.0:1788572563.801126:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801130:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.801133:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.801138:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572563.801146:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572563.801155:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72680 x1875440809974016/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572563.801166:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72680 x1875440809974016/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572648 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.801189:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572563.801200:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801203:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.801568:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801573:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.801577:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801581:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572563.801586:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801590:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801593:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801596:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469a00 00000400:00000200:0.0:1788572563.801599:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.801603:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a731 [1] + 224 00000800:00000001:0.0:1788572563.801607:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801615:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.801618:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.801621:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572563.801624:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572563.801629:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73100 x1875440809974272/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572563.801691:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73100 x1875440809974272/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.801708:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572563.801713:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.801717:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.805486:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.805502:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572563.805506:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.805508:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.805517:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.805524:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572563.805532:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.805538:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.805543:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.805547:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b417469980 00000400:00000200:0.0:1788572563.805552:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572563.805558:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a721 [1] + 224 00000800:00000001:0.0:1788572563.805565:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.805628:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572563.805632:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572563.805657:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572563.805665:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572563.805674:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809974144/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572563.805685:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec71f80 x1875440809974144/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572563.805706:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572563.805715:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572563.805718:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572567.618602:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307706593, at 4307706624 00000100:00000010:0.0:1788572567.618615:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215a20. 00000800:00000200:0.0:1788572567.809824:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572567.809834:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572567.809838:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:0.0:1788572568.769855:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572568.769858:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572568.769861:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572572.739212:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307711712, at 4307711745 00000100:00000010:0.0:1788572572.739222:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215cc0. 00000020:00000001:0.0:1788572573.891104:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572573.891108:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572573.891113:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572574.038007:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572574.038016:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498800. 00000400:00000200:0.0:1788572574.038023:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572574.038030:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572574.038035:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1440 00000400:00000010:0.0:1788572574.038038:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1440. 00000100:00000001:0.0:1788572574.038042:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572574.038050:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6ad80 x1875440809975424/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572574.038061:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572574.038064:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440809975424/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572590 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572574.038070:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572574.038072:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572574.046961:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572574.046977:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572574.046981:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572574.046983:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572574.046991:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572574.046997:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572574.047005:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572574.047020:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572574.047025:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572574.047028:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b417469d80 00000400:00000200:0.0:1788572574.047033:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572574.047038:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a771 [1] + 224 00000800:00000001:0.0:1788572574.047045:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572574.047109:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572574.047113:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572574.047117:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572574.047124:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572574.047131:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68000 x1875440809975168/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572659 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572574.047141:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed68000 x1875440809975168/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572659 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572574.047156:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572574.047164:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572574.047167:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572577.858953:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307716830, at 4307716865 00000100:00000010:0.0:1788572577.858960:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6e40. 00000020:00000001:0.0:1788572579.009934:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572579.009939:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572579.009941:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:0.0:1788572579.169064:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169080:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572579.169084:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169086:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572579.169094:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169101:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572579.169109:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169112:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169117:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169121:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b41746a080 00000400:00000200:0.0:1788572579.169133:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169152:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a791 [1] + 224 00000800:00000001:0.0:1788572579.169160:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169164:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572579.169167:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572579.169171:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572579.169178:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572579.169187:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809975936/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572664 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572579.169198:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6a680 x1875440809975936/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572664 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572579.169219:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572579.169227:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169230:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572579.169232:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169234:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572579.169238:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169241:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572579.169246:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169248:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169251:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169254:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b41746a000 00000400:00000200:0.0:1788572579.169257:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572579.169260:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a7a1 [1] + 224 00000800:00000001:0.0:1788572579.169264:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169319:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572579.169322:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572579.169326:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572579.169329:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572579.169334:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a300 x1875440809975808/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572664 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572579.169356:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6a300 x1875440809975808/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572664 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572579.169364:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572579.169369:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572579.169372:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572582.977821:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307721956, at 4307721984 00000100:00000010:0.0:1788572582.977845:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215de0. 00000020:00000001:0.0:1788572584.129919:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572584.129923:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572584.129926:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572584.260380:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572584.260385:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272086:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572584.272090:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572584.272096:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572584.272098:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572584.272102:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2ec8. 00000400:00000200:0.0:1788572584.272106:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3aa0 00000400:00000010:0.0:1788572584.272109:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3aa0. 00000100:00000001:0.0:1788572584.272111:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572584.272116:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440809976320/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572584.272124:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440809976320/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272129:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572584.272131:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272133:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572584.272136:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572584.272138:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572584.272139:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572584.272142:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572584.272148:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=149 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572584.272152:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=149 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572584.272160:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572584.272163:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572584.272165:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572584.272168:0:2403:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272170:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572584.272176:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440809976320/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272182:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572584.272185:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572584.272187:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572584.272190:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440809976320/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572584.272197:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:874b6a47-e66d-4b10-8108-059b044e1170:2403:1875440809976320:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572584.272203:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572584.272204:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572584.272205:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f27e8. 00000400:00000200:0.0:1788572584.272207:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3880 00000400:00000010:0.0:1788572584.272209:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3880. 00000100:00000001:0.0:1788572584.272211:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572584.272213:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6a00 x1875440809976576/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572584.272220:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6a00 x1875440809976576/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272225:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572584.272226:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272228:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572584.272229:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572584.272230:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572584.272231:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572584.272233:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572584.272238:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=148 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572584.272241:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=59 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572584.272247:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572584.272248:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572584.272250:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572584.272253:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272256:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572584.272258:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869947 00000100:00000001:0.0:1788572584.272260:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572584.272261:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572584.272265:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6a00 x1875440809976576/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272272:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572584.272274:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572584.272276:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572584.272279:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6a00 x1875440809976576/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572584.272286:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809976576:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572584.272291:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572584.272292:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572584.272293:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2cb8. 00000400:00000200:0.0:1788572584.272296:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572584.272297:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572584.272299:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572584.272302:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440809976832/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572584.272309:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440809976832/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272314:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572584.272315:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272317:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572584.272318:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572584.272319:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572584.272321:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572584.272333:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572584.272338:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=148 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572584.272341:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=121 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572584.272343:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572584.272345:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572584.272346:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572584.272348:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272351:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572584.272352:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: skip recheck: last_committed 4295010076 00000100:00000001:0.0:1788572584.272354:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572584.272355:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572584.272359:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440809976832/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272365:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572584.272367:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572584.272368:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572584.272371:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440809976832/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572584.272377:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809976832:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572584.272383:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572584.272384:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572584.272386:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f28f0. 00000400:00000200:0.0:1788572584.272389:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788572584.272390:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788572584.272393:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572584.272395:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4e00 x1875440809976448/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572584.272402:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4e00 x1875440809976448/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272407:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572584.272409:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272414:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572584.272416:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572584.272417:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572584.272418:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572584.272419:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572584.272423:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=148 v=60 (61 61 61 61) 00000100:00001000:0.0:1788572584.272426:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=30 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572584.272428:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572584.272429:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572584.272430:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572584.272432:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272433:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572584.272435:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572584.272436:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572584.272437:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572584.272440:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4e00 x1875440809976448/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272446:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572584.272448:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572584.272449:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572584.272452:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4e00 x1875440809976448/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572584.272460:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809976448:192.168.202.148@tcp:400:kworker.0 00000400:00000001:0.0:1788572584.272465:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572584.272466:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572584.272467:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f25d8. 00000400:00000200:0.0:1788572584.272481:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572584.272482:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572584.272485:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572584.272487:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440809976704/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572584.272494:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440809976704/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272507:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572584.272509:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272511:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572584.272512:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572584.272513:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572584.272515:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572584.272517:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572584.272521:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=148 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572584.272524:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=49 v=5 (1 1 2 0) 00000100:00000001:0.0:1788572584.272526:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572584.272527:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572584.272528:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572584.272530:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272533:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572584.272534:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572584.272536:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572584.272537:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572584.272541:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440809976704/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272547:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572584.272549:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572584.272550:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572584.272554:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440809976704/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572584.272561:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809976704:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572584.272565:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272568:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572584.272572:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440809976320/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272578:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572584.272581:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572584.272592:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708d35800. 02000000:00000001:0.0:1788572584.272595:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572584.272597:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572584.272599:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572584.272601:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572584.272604:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9000. 02000000:00000001:0.0:1788572584.272609:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572584.272611:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572584.272614:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788572584.272617:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572584.272618:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272620:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572584.272624:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6a00 x1875440809976576/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272630:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572584.272632:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572584.272633:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708d35600. 02000000:00000001:0.0:1788572584.272660:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572584.272661:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572584.272662:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572584.272664:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572584.272666:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9200. 02000000:00000001:0.0:1788572584.272668:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572584.272669:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572584.272670:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6a00. 00000100:00000001:0.0:1788572584.272672:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572584.272673:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272674:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572584.272678:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440809976832/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272683:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572584.272684:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572584.272685:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708d34200. 02000000:00000001:0.0:1788572584.272688:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572584.272689:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572584.272690:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572584.272698:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572584.272700:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9800. 02000000:00000001:0.0:1788572584.272702:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572584.272703:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572584.272704:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788572584.272706:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572584.272707:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272709:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572584.272712:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4e00 x1875440809976448/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572669 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272718:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572584.272719:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572584.272720:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708d34400. 02000000:00000001:0.0:1788572584.272722:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572584.272723:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572584.272724:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572584.272725:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572584.272727:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9900. 02000000:00000001:0.0:1788572584.272728:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572584.272729:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572584.272730:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4e00. 00000100:00000001:0.0:1788572584.272733:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572584.272733:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272735:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572584.272738:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440809976704/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572600 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572584.272743:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572584.272744:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572584.272745:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3708d35c00. 02000000:00000001:0.0:1788572584.272747:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572584.272748:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572584.272749:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572584.272750:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572584.272752:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9400. 02000000:00000001:0.0:1788572584.272753:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572584.272754:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572584.272761:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788572584.272763:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572584.272764:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272767:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572584.272774:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572584.272776:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572584.272778:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572584.272781:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572588.097869:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307727071, at 4307727104 00000100:00000010:0.0:1788572588.097874:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788572589.250137:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572589.250141:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572589.250144:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1788572590.762462:0:265462:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788572591.367102:0:265510:0:(debug.c:305:cfs_str2mask()) Process entered 00000100:00000040:0.0:1788572593.219917:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307732198, at 4307732226 00000100:00000010:0.0:1788572593.219933:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215660. 00000020:00000001:0.0:1788572594.371705:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572594.371708:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572594.371710:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572594.499334:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572594.499338:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5200. 00000400:00000200:0.0:1788572594.499343:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572594.499358:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572594.499362:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1cc0 00000400:00000010:0.0:1788572594.499364:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1cc0. 00000100:00000001:0.0:1788572594.499367:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572594.499372:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70000 x1875440809977984/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572594.499380:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572594.499383:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440809977984/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572610 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572594.499389:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572594.499391:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572594.501108:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572594.501119:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572594.501122:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572594.501124:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572594.501129:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572594.501134:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572594.501141:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572594.501144:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572594.501148:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572594.501152:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b41746a700 00000400:00000200:0.0:1788572594.501156:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572594.501160:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a829 [1] + 224 00000800:00000001:0.0:1788572594.501167:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572594.501171:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572594.501174:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572594.501177:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572594.501182:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572594.501187:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73800 x1875440809977600/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572610 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572594.501196:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec73800 x1875440809977600/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572594.501214:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572594.501226:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572594.501229:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572595.862126:0:265945:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.936294:0:266291:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.936300:0:266291:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.936303:0:266291:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572597.936306:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabba8 00000080:00200000:0.0:1788572597.936311:0:266291:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572597.936318:0:266291:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c21c00. 00800000:00000002:0.0:1788572597.936325:0:266291:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572597.936330:0:266291:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572597.936335:0:266291:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936343:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabac8 00800000:00000002:0.0:1788572597.936345:0:266291:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936349:0:266291:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572597.936352:0:266291:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572597.936355:0:266291:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572597.936357:0:266291:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572597.936360:0:266291:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572597.936362:0:266291:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936365:0:266291:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936367:0:266291:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936370:0:266291:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572597.936374:0:266291:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936376:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dab998 00000002:00010000:0.0:1788572597.936378:0:266291:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572597.936382:0:266291:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572597.936384:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.936387:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.936395:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 2 00010000:00000001:0.0:1788572597.936398:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572597.936403:0:266291:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572597.936407:0:266291:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572597.936412:0:266291:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 02000000:00000001:0.0:1788572597.936423:0:266291:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572597.936429:0:266291:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572597.936432:0:266291:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000002:00000001:0.0:1788572597.936438:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572597.936440:0:266291:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572597.936442:0:266291:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936444:0:266291:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936446:0:266291:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.936455:0:266291:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572597.936458:0:266291:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572597.936462:0:266291:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572597.936465:0:266291:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c21c00. 00000080:00000001:0.0:1788572597.936471:0:266291:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572597.936473:0:266291:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936476:0:266291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572597.936480:0:266291:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x144158609316886d 00800000:00000001:0.0:1788572597.936483:0:266291:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572597.936484:0:266291:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572597.936486:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936489:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936492:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936494:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936497:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936500:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936501:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572597.936502:0:266291:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.936504:0:266291:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936506:0:266291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316886d from it ffffb95bc0dabbe0 00010000:00000001:0.0:1788572597.936509:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936510:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936511:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936512:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936514:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936516:0:266291:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572597.936519:0:266291:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00000001:0.0:1788572597.936525:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936526:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.936527:0:266291:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572597.936529:0:266291:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572597.936531:0:266291:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00000001:0.0:1788572597.936547:0:266291:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572597.936550:0:266291:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572597.936552:0:266291:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572597.936556:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936557:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.936558:0:266291:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936562:0:266291:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572597.936564:0:266291:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936566:0:266291:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936568:0:266291:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936570:0:266291:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572597.936571:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936573:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572597.936576:0:266291:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572597.936578:0:266291:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.936580:0:266291:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572597.936585:0:266291:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572597.936594:0:266291:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f171d40. 00000020:00000001:0.0:1788572597.936597:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabd98 00000080:00200000:0.0:1788572597.936599:0:266291:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d801b.sanity, dir=[0x200000007:0x1:0x0](ffff9c372dcc6c08), flags=1536 00000080:00000001:0.0:1788572597.936602:0:266291:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.936603:0:266291:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572597.936605:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabca8 00000080:00200000:0.0:1788572597.936606:0:266291:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572597.936610:0:266291:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c21c00. 00800000:00000002:0.0:1788572597.936613:0:266291:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572597.936615:0:266291:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572597.936617:0:266291:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936618:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabbc8 00800000:00000002:0.0:1788572597.936620:0:266291:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936623:0:266291:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572597.936624:0:266291:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572597.936625:0:266291:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572597.936627:0:266291:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572597.936628:0:266291:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572597.936658:0:266291:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936661:0:266291:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936663:0:266291:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936666:0:266291:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572597.936669:0:266291:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936672:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0daba98 00000002:00010000:0.0:1788572597.936674:0:266291:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572597.936677:0:266291:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572597.936678:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.936680:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.936683:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 2 00010000:00000001:0.0:1788572597.936685:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572597.936688:0:266291:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572597.936691:0:266291:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572597.936694:0:266291:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 02000000:00000001:0.0:1788572597.936702:0:266291:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572597.936705:0:266291:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572597.936708:0:266291:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000002:00000001:0.0:1788572597.936714:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572597.936716:0:266291:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572597.936717:0:266291:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936719:0:266291:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936721:0:266291:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.936723:0:266291:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572597.936725:0:266291:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572597.936727:0:266291:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572597.936729:0:266291:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c21c00. 00000080:00000001:0.0:1788572597.936734:0:266291:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572597.936735:0:266291:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936738:0:266291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572597.936746:0:266291:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x144158609316886d 00800000:00000001:0.0:1788572597.936749:0:266291:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572597.936751:0:266291:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572597.936753:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936754:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936756:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936758:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936761:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936763:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936765:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572597.936766:0:266291:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.936768:0:266291:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936770:0:266291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316886d from it ffffb95bc0dabce0 00010000:00000001:0.0:1788572597.936776:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936777:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936778:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936780:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936781:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936783:0:266291:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572597.936786:0:266291:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00000001:0.0:1788572597.936793:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936794:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.936795:0:266291:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572597.936797:0:266291:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572597.936799:0:266291:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00000001:0.0:1788572597.936805:0:266291:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572597.936807:0:266291:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572597.936808:0:266291:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572597.936811:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936812:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.936819:0:266291:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936821:0:266291:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572597.936822:0:266291:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936824:0:266291:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936826:0:266291:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.936828:0:266291:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572597.936830:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936831:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572597.936833:0:266291:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572597.936834:0:266291:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.936836:0:266291:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 3 00000080:00000001:0.0:1788572597.936839:0:266291:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.936843:0:266291:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.936844:0:266291:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572597.936845:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabd90 00000080:00200000:0.0:1788572597.936847:0:266291:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572597.936851:0:266291:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c21c00. 00800000:00000002:0.0:1788572597.936855:0:266291:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572597.936857:0:266291:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572597.936859:0:266291:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936861:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabcb0 00800000:00000002:0.0:1788572597.936862:0:266291:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936865:0:266291:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572597.936866:0:266291:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572597.936867:0:266291:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572597.936869:0:266291:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572597.936870:0:266291:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572597.936872:0:266291:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936873:0:266291:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.936875:0:266291:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.936877:0:266291:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572597.936880:0:266291:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.936881:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabb80 00000002:00010000:0.0:1788572597.936883:0:266291:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572597.936886:0:266291:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572597.936887:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.936889:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.936897:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 2 00010000:00000001:0.0:1788572597.936899:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572597.936902:0:266291:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572597.936904:0:266291:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572597.936907:0:266291:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 02000000:00000001:0.0:1788572597.936914:0:266291:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572597.936916:0:266291:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572597.936919:0:266291:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000002:00000001:0.0:1788572597.936925:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572597.936927:0:266291:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572597.936929:0:266291:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936931:0:266291:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572597.936933:0:266291:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.936935:0:266291:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572597.936952:0:266291:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572597.936954:0:266291:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572597.936956:0:266291:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c21c00. 00000080:00000001:0.0:1788572597.936960:0:266291:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572597.936961:0:266291:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936963:0:266291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572597.936965:0:266291:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x144158609316886d 00800000:00000001:0.0:1788572597.936968:0:266291:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572597.936969:0:266291:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572597.936971:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936972:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936973:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936975:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936977:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.936979:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.936986:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572597.936988:0:266291:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.936989:0:266291:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572597.936992:0:266291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316886d from it ffffb95bc0dabdc8 00010000:00000001:0.0:1788572597.936994:0:266291:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572597.936995:0:266291:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572597.936997:0:266291:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3718424a00 refcount=3 00000020:00000001:0.0:1788572597.936999:0:266291:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.937001:0:266291:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359176972800 : -109714532578816 : ffff9c3718424a00) 00010000:00000001:0.0:1788572597.937003:0:266291:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572597.937006:0:266291:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00000001:0.0:1788572597.937012:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.937014:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.937015:0:266291:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572597.937017:0:266291:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572597.937020:0:266291:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00000001:0.0:1788572597.937025:0:266291:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572597.937028:0:266291:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572597.937029:0:266291:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572597.937032:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.937034:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.937035:0:266291:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937037:0:266291:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572597.937039:0:266291:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937041:0:266291:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937043:0:266291:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937045:0:266291:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572597.937046:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.937047:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572597.937049:0:266291:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572597.937050:0:266291:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572597.937052:0:266291:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 3 00000080:00000001:0.0:1788572597.937065:0:266291:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.937069:0:266291:0:(namei.c:2169:do_mkdir()) Process entered 00000020:00000001:0.0:1788572597.937071:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabda0 00000080:00200000:0.0:1788572597.937073:0:266291:0:(namei.c:2172:do_mkdir()) VFS Op:name=d801b.sanity, dir=[0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00000001:0.0:1788572597.937077:0:266291:0:(namei.c:1968:ll_new_node()) Process entered 00000080:00000001:0.0:1788572597.937079:0:266291:0:(namei.c:1778:ll_new_node_prepare()) Process entered 00000080:00000010:0.0:1788572597.937082:0:266291:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c21c00. 00800000:00000002:0.0:1788572597.937085:0:266291:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000010:0.0:1788572597.937092:0:266291:0:(namei.c:1796:ll_new_node_prepare()) kmalloced '(lum)': 48 at ffff9c372e9a9f40. 00000080:00000001:0.0:1788572597.937095:0:266291:0:(namei.c:1884:ll_new_node_prepare()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.937099:0:266291:0:(lmv_obd.c:2310:lmv_create()) Process entered 00800000:00000001:0.0:1788572597.937101:0:266291:0:(lmv_obd.c:2248:lmv_locate_tgt_create()) Process entered 00800000:00000001:0.0:1788572597.937104:0:266291:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572597.937105:0:266291:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572597.937106:0:266291:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572597.937108:0:266291:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572597.937110:0:266291:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.937112:0:266291:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.937114:0:266291:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.937119:0:266291:0:(lmv_obd.c:1705:lmv_locate_tgt_qos()) Process entered 00020000:00000001:0.0:1788572597.937122:0:266291:0:(lu_tgt_descs.c:432:ltd_qos_penalties_calc()) Process entered 00020000:00000001:0.0:1788572597.937124:0:266291:0:(lu_tgt_descs.c:435:ltd_qos_penalties_calc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572597.937127:0:266291:0:(lu_tgt_descs.c:571:ltd_qos_penalties_calc()) Process leaving (rc=0 : 0 : 0) 00020000:00001000:0.0:1788572597.937130:0:266291:0:(lu_tgt_descs.c:211:lu_tgt_qos_weight_calc()) ltq_penalty: 2992560 lsq_penalty: 0 tgt_bavail: 34921728 00020000:00001000:0.0:1788572597.937133:0:266291:0:(lu_tgt_descs.c:211:lu_tgt_qos_weight_calc()) ltq_penalty: 7957046 lsq_penalty: 0 tgt_bavail: 78346300 00020000:00000001:0.0:1788572597.937137:0:266291:0:(lu_tgt_descs.c:594:ltd_qos_update()) Process entered 00020000:00001000:0.0:1788572597.937138:0:266291:0:(lu_tgt_descs.c:619:ltd_qos_update()) ltq_penalty: 8589756 per_obj: 3546738 tgt_count: 2 00020000:00001000:0.0:1788572597.937141:0:266291:0:(lu_tgt_descs.c:624:ltd_qos_update()) lsq_penalty: 5744727 per_obj: 5744727 srv_count: 1 00020000:00001000:0.0:1788572597.937143:0:266291:0:(lu_tgt_descs.c:211:lu_tgt_qos_weight_calc()) ltq_penalty: 5043018 lsq_penalty: 0 tgt_bavail: 34921728 00020000:00001000:0.0:1788572597.937146:0:266291:0:(lu_tgt_descs.c:661:ltd_qos_update()) recalc tgt 0 usable=0 bavail=8832 ffree=3954 tgtppo=3463 tgtp=4924 svrppo=5610 svrp=0 wt=29178 00020000:00001000:0.0:1788572597.937149:0:266291:0:(lu_tgt_descs.c:211:lu_tgt_qos_weight_calc()) ltq_penalty: 0 lsq_penalty: 0 tgt_bavail: 78346300 00020000:00001000:0.0:1788572597.937151:0:266291:0:(lu_tgt_descs.c:661:ltd_qos_update()) recalc tgt 1 usable=1 bavail=19685 ffree=3980 tgtppo=7770 tgtp=0 svrppo=5610 svrp=0 wt=76510 00020000:00000001:0.0:1788572597.937155:0:266291:0:(lu_tgt_descs.c:664:ltd_qos_update()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.937157:0:266291:0:(lmv_obd.c:1768:lmv_locate_tgt_qos()) Process leaving via unlock (rc=18446634359068665856 : -109714640885760 : 0xffff9c3711cda800) 00800000:00000001:0.0:1788572597.937167:0:266291:0:(lmv_obd.c:2294:lmv_locate_tgt_create()) Process leaving (rc=18446634359068665856 : -109714640885760 : ffff9c3711cda800) 00800000:00000001:0.0:1788572597.937170:0:266291:0:(lmv_obd.c:1208:lmv_fid_alloc()) Process entered 00800000:00000001:0.0:1788572597.937172:0:266291:0:(obd_class.h:931:obd_fid_alloc()) Process entered 00000002:00000001:0.0:1788572597.937175:0:266291:0:(mdc_request.c:2962:mdc_fid_alloc()) Process entered 40000000:00000001:0.0:1788572597.937178:0:266291:0:(fid_request.c:299:seq_client_alloc_fid()) Process entered 40000000:00000040:0.0:1788572597.937181:0:266291:0:(fid_request.c:336:seq_client_alloc_fid()) cli-cli-lustre-MDT0000-mdc-ffff9c3711155800: Allocated FID [0x200002b17:0xdf9:0x0] 40000000:00000001:0.0:1788572597.937184:0:266291:0:(fid_request.c:340:seq_client_alloc_fid()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.937186:0:266291:0:(mdc_request.c:2968:mdc_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.937187:0:266291:0:(obd_class.h:944:obd_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572597.937189:0:266291:0:(lmv_obd.c:1232:lmv_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572597.937192:0:266291:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc0dabc08 00800000:00000002:0.0:1788572597.937194:0:266291:0:(lmv_obd.c:2351:lmv_create()) CREATE name 'd801b.sanity' [0x200002b17:0xdf9:0x0] on [0x200000007:0x1:0x0] -> mds #0, layout version 1 00000002:00000001:0.0:1788572597.937198:0:266291:0:(mdc_reint.c:154:mdc_create()) Process entered 00000002:00000001:0.0:1788572597.937201:0:266291:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572597.937204:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 2 00010000:00000001:0.0:1788572597.937206:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572597.937209:0:266291:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572597.937212:0:266291:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572597.937215:0:266291:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00000001:0.0:1788572597.937223:0:266291:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572597.937226:0:266291:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572597.937228:0:266291:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572597.937230:0:266291:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200000007:0x1:0x0] mode (null) 00000002:00000001:0.0:1788572597.937233:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.937234:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.937236:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 3 00010000:00000001:0.0:1788572597.937238:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572597.937241:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.937243:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.937244:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.937246:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 3 00010000:00000001:0.0:1788572597.937248:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572597.937251:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.937260:0:266291:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.937261:0:266291:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572597.937264:0:266291:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37188c6e00 count: 3 00010000:00000001:0.0:1788572597.937267:0:266291:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572597.937269:0:266291:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.937271:0:266291:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000002:0.0:1788572597.937274:0:266291:0:(namei.c:327:ll_lock_cancel_bits()) invalidating inode [0x200000007:0x1:0x0] lli = ffff9c372dcc6b40, pfid = [0x0:0x0:0x0] 00000080:00000001:0.0:1788572597.937278:0:266291:0:(namei.c:165:ll_prune_negative_children()) Process entered 00000080:00000001:0.0:1788572597.937281:0:266291:0:(namei.c:196:ll_prune_negative_children()) Process leaving 00000080:00000001:0.0:1788572597.937283:0:266291:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572597.937285:0:266291:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937289:0:266291:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572597.937293:0:266291:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572597.937294:0:266291:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572597.937297:0:266291:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572597.937299:0:266291:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788572597.937302:0:266291:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c3718424a00 with handle 0x144158609316886d from hash 00010000:00000001:0.0:1788572597.937305:0:266291:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572597.937306:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.937308:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.937309:0:266291:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572597.937312:0:266291:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572597.937313:0:266291:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572597.937317:0:266291:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572597.937319:0:266291:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572597.937321:0:266291:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788572597.937328:0:266291:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572597.937332:0:266291:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572597.937338:0:266291:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572597.937340:0:266291:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937343:0:266291:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572597.937347:0:266291:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572597.937350:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.937352:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.937353:0:266291:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.937356:0:266291:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572597.937363:0:266291:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572597.937366:0:266291:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572597.937372:0:266291:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572597.937378:0:266291:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937381:0:266291:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572597.937383:0:266291:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572597.937389:0:266291:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1788572597.937396:0:266291:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572597.937398:0:266291:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572597.937400:0:266291:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572597.937402:0:266291:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3718424a00/0x144158609316886d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd9894672d99a2c4d expref: -99 pid: 262642 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00000040:0.0:1788572597.937410:0:266291:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37188c6e00 count: 0 00010000:00000010:0.0:1788572597.937415:0:266291:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff9c37320cf880. 00010000:00000001:0.0:1788572597.937423:0:266291:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.937425:0:266291:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937427:0:266291:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572597.937428:0:266291:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.937431:0:266291:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000020:00100000:0.0:1788572597.937441:0:266291:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788572597.937454:0:266291:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b7480 x1875440809978368/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/840 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572597.937462:0:266291:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572597.937464:0:266291:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572597.937470:0:266291:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a000. 00000100:00000001:0.0:1788572597.937472:0:266291:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040024576 : -109714669527040 : ffff9c371018a000) 00000100:00000001:0.0:1788572597.937475:0:266291:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572597.937476:0:266291:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000001:0.0:1788572597.937480:0:266291:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572597.937484:0:266291:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.937486:0:266291:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572597.937493:0:266291:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572597.937499:0:266291:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440809978368/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/840 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.937506:0:266291:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572597.937509:0:266291:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572597.937513:0:266291:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572597.937515:0:266291:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572597.937518:0:266291:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572597.937521:0:266291:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266291:1875440809978368:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572597.937525:0:266291:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572597.937530:0:266291:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572597.937532:0:266291:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572597.937535:0:266291:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572597.937540:0:266291:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109eb000. 02000000:00000001:0.0:1788572597.937542:0:266291:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572597.937549:0:266291:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572597.937553:0:266291:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8688. 00000400:00000010:0.0:1788572597.937560:0:266291:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572597.937567:0:266291:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809978368, portal 10 00000100:00000001:0.0:1788572597.937569:0:266291:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572597.937570:0:266291:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000040:0.0:1788572597.937577:0:266291:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7480 x1875440809978368/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/840 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.937584:0:266291:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572597.937587:0:266291:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572597.937589:0:266291:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572597.937592:0:266291:0:(niobuf.c:94:ptl_send_buf()) Sending 560 bytes to portal 12, xid 1875440809978368, offset 0 00000400:00000200:0.0:1788572597.937597:0:266291:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572597.937605:0:266291:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.937611:0:266291:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572597.937616:0:266291:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572597.937620:0:266291:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572597.937623:0:266291:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572597.937631:0:266291:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572597.937792:0:266291:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540015:540015:256:4294967295] 192.168.202.148@tcp LPNI seq info [539980:539980:8:4294967295] 00000400:00000200:0.0:1788572597.937798:0:266291:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.937808:0:266291:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572597.937816:0:266291:0:(socklnd_cb.c:931:ksocknal_send()) sending 560 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572597.938259:0:266291:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572597.938269:0:266291:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572597.938277:0:266291:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572597.938281:0:266291:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0400 type 1, nob 656 niov 1 nkiov 1 00000100:00000001:0.0:1788572597.938296:0:266291:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572597.938302:0:266291:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15136 usecs (16 jiffies) 00000400:00000200:0.0:1788572597.938306:0:266291:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572597.938310:0:266291:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307736960 00000100:00000001:0.0:1788572597.938313:0:266291:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307737310 00000100:00000001:0.0:1788572597.938316:0:266291:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307736960 00000100:00000001:0.0:1788572597.938319:0:266291:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307737310 00000100:00000001:0.0:1788572597.938321:0:266291:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572597.938324:0:266291:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.938327:0:266291:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572597.938329:0:266291:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572597.938332:0:266291:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a000 going to sleep for 80 seconds 00000100:00000001:0.0:1788572597.938335:0:266291:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572597.938338:0:266291:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.942367:0:266291:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572597.942375:0:266291:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572597.942377:0:266291:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572597.942381:0:266291:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8688. 00000400:00000200:0.0:1788572597.942385:0:266291:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572597.942388:0:266291:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572597.942392:0:266291:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572597.942398:0:266291:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7480 x1875440809978368/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/840 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572597.942425:0:266291:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7480 x1875440809978368/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/840 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.942433:0:266291:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572597.942435:0:266291:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572597.942438:0:266291:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572597.942441:0:266291:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572597.942444:0:266291:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572597.942446:0:266291:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572597.942449:0:266291:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572597.942457:0:266291:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=11 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572597.942470:0:266291:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=43 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572597.942474:0:266291:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572597.942476:0:266291:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.942479:0:266291:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572597.942481:0:266291:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.942484:0:266291:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572597.942486:0:266291:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x400017581/0x1] 00000100:00000001:0.0:1788572597.942487:0:266291:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572597.942490:0:266291:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572597.942491:0:266291:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000001:0.0:1788572597.942494:0:266291:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572597.942496:0:266291:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859796 00000100:00000001:0.0:1788572597.942498:0:266291:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572597.942501:0:266291:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572597.942507:0:266291:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7480 x1875440809978368/t25769859799(25769859799) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/448 e 0 to 0 dl 1788572682 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.942515:0:266291:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572597.942518:0:266291:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572597.942520:0:266291:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572597.942525:0:266291:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7480 x1875440809978368/t25769859799(25769859799) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/448 e 0 to 0 dl 1788572682 ref 3 fl Interpret:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572597.942534:0:266291:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266291:1875440809978368:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572597.942547:0:266291:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572597.942551:0:266291:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.942553:0:266291:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572597.942555:0:266291:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.942559:0:266291:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b7480 x1875440809978368/t25769859799(25769859799) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/448 e 0 to 0 dl 1788572682 ref 3 fl Complete:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.942566:0:266291:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572597.942569:0:266291:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a000. 00000100:00000001:0.0:1788572597.942572:0:266291:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572597.942573:0:266291:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.942581:0:266291:0:(mdc_reint.c:296:mdc_create()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.942585:0:266291:0:(lmv_obd.c:2359:lmv_create()) Created - [0x200002b17:0xdf9:0x0] 00800000:00000001:0.0:1788572597.942587:0:266291:0:(lmv_obd.c:2416:lmv_create()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942591:0:266291:0:(namei.c:1904:ll_new_node_finish()) Process entered 00000080:00000002:0.0:1788572597.942594:0:266291:0:(namei.c:1700:ll_update_times()) setting fid [0x200000007:0x1:0x0] mtime from 1788572494 to 1788572597 00000080:00000001:0.0:1788572597.942599:0:266291:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572597.942616:0:266291:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572597.942619:0:266291:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572597.942622:0:266291:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329145 : 144115373145329145 : 200002b17000df9) 00000080:00000001:0.0:1788572597.942625:0:266291:0:(namei.c:113:ll_iget()) Process entered 00000080:00000010:0.0:1788572597.942709:0:266291:0:(super25.c:42:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff9c3708704a40. 00000080:00000001:0.0:1788572597.942724:0:266291:0:(llite_lib.c:3353:ll_read_inode2()) Process entered 00000080:00200000:0.0:1788572597.942727:0:266291:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200002b17:0xdf9:0x0](ffff9c3708704b08) 00000080:00000001:0.0:1788572597.942731:0:266291:0:(llite_lib.c:1957:ll_update_lsm_md()) Process entered 00000080:00000002:0.0:1788572597.942734:0:266291:0:(llite_lib.c:1961:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200002b17:0xdf9:0x0] 00000080:00000001:0.0:1788572597.942736:0:266291:0:(llite_lib.c:1978:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942739:0:266291:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329145 : 144115373145329145 : 200002b17000df9) 00000080:00000001:0.0:1788572597.942742:0:266291:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00000002:0.0:1788572597.942744:0:266291:0:(llite_lib.c:3050:ll_update_inode()) setting ino 144115373145329145 mtime from 0 to 1788572597 00000080:00200000:0.0:1788572597.942749:0:266291:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200002b17:0xdf9:0x0], updating i_size 4096 00000080:00000001:0.0:1788572597.942752:0:266291:0:(llite_lib.c:3382:ll_read_inode2()) Process leaving 00000080:00000001:0.0:1788572597.942755:0:266291:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911552264 : -109714797999352 : ffff9c3708704b08) 00000080:00000001:0.0:1788572597.942757:0:266291:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572597.942760:0:266291:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572597.942769:0:266291:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942775:0:266291:0:(namei.c:1951:ll_new_node_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942777:0:266291:0:(namei.c:1996:ll_new_node()) Process leaving 00000100:00000001:0.0:1788572597.942779:0:266291:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.942784:0:266291:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440809978368/t25769859799(25769859799) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 560/448 e 0 to 0 dl 1788572682 ref 2 fl Complete:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.942791:0:266291:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.942794:0:266291:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572597.942797:0:266291:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c21c00. 00000080:00000010:0.0:1788572597.942803:0:266291:0:(namei.c:2002:ll_new_node()) kfreed 'lum': 48 at ffff9c372e9a9f40. 00000080:00000001:0.0:1788572597.942807:0:266291:0:(namei.c:2004:ll_new_node()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942811:0:266291:0:(namei.c:2180:do_mkdir()) Process leaving via out_tally (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572597.942815:0:266291:0:(namei.c:2224:do_mkdir()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572597.942823:0:266291:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:0.0:1788572597.942829:0:266291:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:0.0:1788572597.949789:0:266295:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.949793:0:266295:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572597.949795:0:266295:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572597.949798:0:266295:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c1fba8 00000080:00200000:0.0:1788572597.949801:0:266295:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572597.949807:0:266295:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20a00. 00800000:00000002:0.0:1788572597.949813:0:266295:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572597.949818:0:266295:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572597.949823:0:266295:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.949825:0:266295:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c1fac8 00800000:00000002:0.0:1788572597.949827:0:266295:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.949831:0:266295:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572597.949834:0:266295:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572597.949836:0:266295:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572597.949838:0:266295:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572597.949840:0:266295:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572597.949841:0:266295:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.949843:0:266295:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572597.949844:0:266295:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572597.949847:0:266295:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572597.949851:0:266295:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572597.949852:0:266295:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1c1f998 00000002:00010000:0.0:1788572597.949859:0:266295:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572597.949861:0:266295:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572597.949863:0:266295:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572597.949865:0:266295:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572597.949870:0:266295:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.949871:0:266295:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.949873:0:266295:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.949874:0:266295:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572597.949876:0:266295:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572597.949881:0:266295:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572597.949883:0:266295:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572597.949888:0:266295:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572597.949892:0:266295:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572597.949897:0:266295:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572597.949898:0:266295:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572597.949899:0:266295:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.949902:0:266295:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572597.949904:0:266295:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572597.949908:0:266295:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572597.949912:0:266295:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572597.949918:0:266295:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.949922:0:266295:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572597.949923:0:266295:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.949924:0:266295:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572597.949925:0:266295:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572597.949926:0:266295:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572597.949929:0:266295:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00010000:00000001:0.0:1788572597.949932:0:266295:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572597.949934:0:266295:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572597.949951:0:266295:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46a00. 00010000:00000010:0.0:1788572597.949955:0:266295:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676180. 00010000:00000001:0.0:1788572597.949958:0:266295:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572597.949962:0:266295:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788572597.949965:0:266295:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572597.949968:0:266295:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x14415860931688cf to hash 00000020:00000001:0.0:1788572597.949970:0:266295:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572597.949976:0:266295:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788572597.949980:0:266295:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788572597.949984:0:266295:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572597.949987:0:266295:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572597.949991:0:266295:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x14415860931688cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 266295 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572597.950001:0:266295:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x14415860931688cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 266295 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572597.950010:0:266295:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x14415860931688cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 266295 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572597.950016:0:266295:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572597.950018:0:266295:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572597.950024:0:266295:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a000. 00000100:00000001:0.0:1788572597.950026:0:266295:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040024576 : -109714669527040 : ffff9c371018a000) 00000100:00000001:0.0:1788572597.950029:0:266295:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572597.950030:0:266295:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000001:0.0:1788572597.950035:0:266295:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572597.950039:0:266295:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.950041:0:266295:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572597.950042:0:266295:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572597.950051:0:266295:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440809978496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.950059:0:266295:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572597.950060:0:266295:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572597.950063:0:266295:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572597.950065:0:266295:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572597.950067:0:266295:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572597.950069:0:266295:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job createmany:5fa795d2-07a0-4260-8bbc-bb3cde572406:266295:1875440809978496:192.168.202.148@tcp:101:createmany.0 00000100:00000001:0.0:1788572597.950072:0:266295:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572597.950077:0:266295:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572597.950082:0:266295:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572597.950084:0:266295:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572597.950089:0:266295:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ee000. 02000000:00000001:0.0:1788572597.950090:0:266295:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572597.950098:0:266295:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572597.950102:0:266295:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8688. 00000400:00000010:0.0:1788572597.950109:0:266295:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572597.950116:0:266295:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809978496, portal 10 00000100:00000001:0.0:1788572597.950118:0:266295:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572597.950119:0:266295:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000040:0.0:1788572597.950126:0:266295:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6300 x1875440809978496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.950133:0:266295:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572597.950136:0:266295:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572597.950140:0:266295:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572597.950143:0:266295:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1875440809978496, offset 0 00000400:00000200:0.0:1788572597.950149:0:266295:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572597.950156:0:266295:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.950162:0:266295:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572597.950166:0:266295:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572597.950169:0:266295:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572597.950172:0:266295:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572597.950174:0:266295:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572597.950177:0:266295:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540016:540016:256:4294967295] 192.168.202.148@tcp LPNI seq info [539981:539981:8:4294967295] 00000400:00000200:0.0:1788572597.950180:0:266295:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.950186:0:266295:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572597.950190:0:266295:0:(socklnd_cb.c:931:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572597.950194:0:266295:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572597.950199:0:266295:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572597.950204:0:266295:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572597.950208:0:266295:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0200 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1788572597.950220:0:266295:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572597.950228:0:266295:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15133 usecs (16 jiffies) 00000400:00000200:0.0:1788572597.950231:0:266295:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572597.950233:0:266295:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307736971 00000100:00000001:0.0:1788572597.950235:0:266295:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307737310 00000100:00000001:0.0:1788572597.950237:0:266295:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307736971 00000100:00000001:0.0:1788572597.950238:0:266295:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307737310 00000100:00000001:0.0:1788572597.950240:0:266295:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572597.950242:0:266295:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.950244:0:266295:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572597.950245:0:266295:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572597.950247:0:266295:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a000 going to sleep for 80 seconds 00000100:00000001:0.0:1788572597.950249:0:266295:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572597.950251:0:266295:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572597.953320:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.953334:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572597.953338:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.953342:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.953350:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.953360:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572597.953369:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.953374:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.953379:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.953383:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 688 into portal 10 MB=0x6a9b41746aa80 00000400:00000200:0.0:1788572597.953389:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.953395:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 688/688 into md 0x52a861 [1] + 224 00000800:00000001:0.0:1788572597.953403:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.953414:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.953417:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.953421:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572597.953428:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572597.953435:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6300 x1875440809978496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572682 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572597.953459:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=688 offset=224 replen=1680 req@ffff9c372d6b6300 x1875440809978496/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.953477:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572597.953486:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.953490:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572597.954924:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572597.954928:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572597.954932:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.954950:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572597.954954:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572597.954956:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572597.954960:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572597.954964:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809978624/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.954973:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.954976:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5f80 x1875440809978624/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572682 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.954982:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.954984:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572597.957596:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.957607:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572597.957610:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.957612:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.957618:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.957625:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572597.957631:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.957662:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.957666:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.957669:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746ab00 00000400:00000200:0.0:1788572597.957673:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.957678:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a871 [1] + 224 00000800:00000001:0.0:1788572597.957690:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.957697:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.957699:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.957703:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572597.957708:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572597.957714:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5f80 x1875440809978624/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572682 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572597.957721:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371f1c5f80 x1875440809978624/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572597.957738:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572597.957746:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.957750:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572597.959197:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572597.959200:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572597.959203:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.959207:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572597.959210:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3990 00000400:00000010:0.0:1788572597.959212:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3990. 00000100:00000001:0.0:1788572597.959214:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572597.959217:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809978880/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.959223:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.959225:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5f80 x1875440809978880/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.959229:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.959231:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572597.982926:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.982958:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572597.982963:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.982965:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.982975:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.982982:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572597.983005:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.983010:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.983016:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.983020:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746ac00 00000400:00000200:0.0:1788572597.983024:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.983030:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a881 [1] + 224 00000800:00000001:0.0:1788572597.983037:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.983044:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.983048:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.983052:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572597.983059:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572597.983077:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5f80 x1875440809978880/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572597.983088:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=576 req@ffff9c371f1c5f80 x1875440809978880/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.983110:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572597.983117:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.983121:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572597.984556:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572597.984561:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572597.984565:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.984570:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572597.984574:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f36e8 00000400:00000010:0.0:1788572597.984575:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f36e8. 00000100:00000001:0.0:1788572597.984578:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572597.984581:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4e00 x1875440809979136/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.984587:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.984589:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4e00 x1875440809979136/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572682 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.984594:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.984599:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572597.988544:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.988559:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572597.988563:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.988566:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.988574:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.988581:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572597.988589:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.988594:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.988598:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.988612:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746ad00 00000400:00000200:0.0:1788572597.988616:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572597.988622:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a891 [1] + 224 00000800:00000001:0.0:1788572597.988630:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.988680:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572597.988684:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.988689:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572597.988696:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572597.988704:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4e00 x1875440809979136/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572682 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572597.988715:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=840 req@ffff9c371f1c4e00 x1875440809979136/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.988733:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572597.988741:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572597.988744:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572597.990092:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572597.990097:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788572597.990101:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572597.990106:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572597.990109:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572597.990111:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572597.990114:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572597.990119:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4380 x1875440809979392/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.990143:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572597.990147:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c4380 x1875440809979392/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572597.990153:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572597.990155:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.010994:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.011013:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.011018:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.011021:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.011029:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.011038:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.011046:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.011052:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.011057:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.011061:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746ae00 00000400:00000200:0.0:1788572598.011066:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.011072:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a8a1 [1] + 224 00000800:00000001:0.0:1788572598.011079:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.011087:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.011090:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.011095:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.011103:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.011111:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4380 x1875440809979392/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.011122:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=576 req@ffff9c371f1c4380 x1875440809979392/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572682 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.011139:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.011147:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.011150:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.014341:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.014351:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788572598.014355:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.014360:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.014364:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572598.014365:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572598.014368:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.014371:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6a00 x1875440809979648/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.014379:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.014381:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6a00 x1875440809979648/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.014386:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.014387:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.017974:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.017990:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.017995:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.017997:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.018006:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.018014:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.018021:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.018025:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.018029:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.018032:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746af00 00000400:00000200:0.0:1788572598.018036:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.018040:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a8b1 [1] + 224 00000800:00000001:0.0:1788572598.018045:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.018050:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.018052:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.018055:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.018060:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.018066:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6a00 x1875440809979648/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.018073:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=840 req@ffff9c371f1c6a00 x1875440809979648/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.018107:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.018116:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.018119:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.019283:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.019287:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572598.019289:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.019293:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.019295:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572598.019297:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572598.019299:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.019303:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7800 x1875440809979904/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.019309:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.019311:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7800 x1875440809979904/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.019315:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.019316:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.022559:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.022571:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.022574:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.022576:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.022582:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.022587:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.022593:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.022597:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.022600:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.022603:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746b000 00000400:00000200:0.0:1788572598.022606:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.022611:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a8c1 [1] + 224 00000800:00000001:0.0:1788572598.022615:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.022621:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.022633:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.022671:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.022678:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.022685:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7800 x1875440809979904/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.022694:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=840 req@ffff9c371f1c7800 x1875440809979904/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/840 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.022706:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.022712:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.022715:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.023789:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.023794:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572598.023798:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.023803:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.023806:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788572598.023808:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788572598.023811:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.023815:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809980160/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.023823:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.023826:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c6300 x1875440809980160/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.023832:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.023834:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.038437:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.038454:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.038458:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.038460:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.038468:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.038476:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.038485:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.038490:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.038529:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.038535:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746b100 00000400:00000200:0.0:1788572598.038539:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.038545:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52a8d1 [1] + 224 00000800:00000001:0.0:1788572598.038552:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.038559:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.038562:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.038566:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.038573:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.038581:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809980160/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.038591:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=576 req@ffff9c371f1c6300 x1875440809980160/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'createmany.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.038612:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.038620:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.038622:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572598.337833:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307737310, at 4307737344 00000100:00000010:0.0:1788572598.337839:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6d80. 00000800:00000200:0.0:1788572598.615433:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.615438:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5100. 00000400:00000200:0.0:1788572598.615443:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.615449:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.615455:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1a18 00000400:00000010:0.0:1788572598.615457:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1a18. 00000100:00000001:0.0:1788572598.615461:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.615465:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69f80 x1875440809980288/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.615472:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.615474:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed69f80 x1875440809980288/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.615479:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.615485:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.625112:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.625126:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.625129:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.625130:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.625138:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.625146:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.625154:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.625159:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.625164:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.625168:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746b180 00000400:00000200:0.0:1788572598.625173:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.625179:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a8e1 [1] + 224 00000800:00000001:0.0:1788572598.625186:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.625191:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.625193:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.625197:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.625209:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.625218:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69f80 x1875440809980288/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.625226:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371ed69f80 x1875440809980288/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.625237:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.625243:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.625246:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.626451:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.626454:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5e00. 00000400:00000200:0.0:1788572598.626457:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.626460:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.626464:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1e58 00000400:00000010:0.0:1788572598.626466:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1e58. 00000100:00000001:0.0:1788572598.626469:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.626473:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809980416/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.626490:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.626493:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6aa00 x1875440809980416/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.626499:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.626501:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.629554:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.629571:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.629576:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.629579:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.629587:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.629595:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.629604:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.629609:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.629614:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.629619:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746b200 00000400:00000200:0.0:1788572598.629624:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.629630:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a8e9 [1] + 224 00000800:00000001:0.0:1788572598.629673:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.629682:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.629685:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.629691:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.629712:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.629720:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809980416/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.629733:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1152 req@ffff9c371ed6aa00 x1875440809980416/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1152 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.629749:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.629757:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.629760:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.631128:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.631133:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5c00. 00000400:00000200:0.0:1788572598.631142:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.631147:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.631151:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1330 00000400:00000010:0.0:1788572598.631153:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1330. 00000100:00000001:0.0:1788572598.631157:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.631162:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6aa00 x1875440809980672/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 648/66320 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.631171:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.631175:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6aa00 x1875440809980672/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 648/66320 e 0 to 0 dl 1788572683 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.631182:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.631184:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.636557:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.636576:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.636580:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.636583:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.636591:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.636599:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.636608:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.636612:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.636617:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.636621:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b41746b300 00000400:00000200:0.0:1788572598.636626:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.636632:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a8f1 [32] + 224 00000800:00000001:0.0:1788572598.636674:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.636680:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.636683:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.636688:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.636695:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.636702:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6aa00 x1875440809980672/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 648/66320 e 0 to 0 dl 1788572683 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.636718:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66320 req@ffff9c371ed6aa00 x1875440809980672/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 648/66320 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.636751:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.636760:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.636763:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.638708:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.638713:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5f00. 00000400:00000200:0.0:1788572598.638717:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.638722:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.638726:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1f68 00000400:00000010:0.0:1788572598.638728:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1f68. 00000100:00000001:0.0:1788572598.638732:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.638736:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69f80 x1875440809980928/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.638745:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.638748:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed69f80 x1875440809980928/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.638754:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.638756:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.641062:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.641076:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.641080:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.641083:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.641090:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.641097:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.641105:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.641110:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.641115:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.641120:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746b400 00000400:00000200:0.0:1788572598.641125:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.641131:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a8f9 [1] + 224 00000800:00000001:0.0:1788572598.641138:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.641146:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.641163:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.641168:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.641174:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.641183:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69f80 x1875440809980928/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.641194:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=456 offset=224 replen=512 req@ffff9c371ed69f80 x1875440809980928/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.641266:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.641274:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.641278:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.642220:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.642225:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5f00. 00000400:00000200:0.0:1788572598.642228:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.642233:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.642237:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1c38 00000400:00000010:0.0:1788572598.642239:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1c38. 00000100:00000001:0.0:1788572598.642242:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.642246:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a680 x1875440809981056/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.642255:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.642258:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6a680 x1875440809981056/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.642272:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.642274:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.644435:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.644449:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.644453:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.644455:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.644463:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.644469:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.644477:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.644482:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.644499:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.644503:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 10 MB=0x6a9b41746b480 00000400:00000200:0.0:1788572598.644507:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.644513:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a901 [1] + 224 00000800:00000001:0.0:1788572598.644518:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.644525:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.644528:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.644532:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.644537:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.644544:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809981056/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.644566:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=472 req@ffff9c371ed6a680 x1875440809981056/t0(0) o101->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/472 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.644581:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.644588:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.644591:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.646046:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.646051:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498e00. 00000400:00000200:0.0:1788572598.646055:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.646059:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.646073:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1330 00000400:00000010:0.0:1788572598.646075:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1330. 00000100:00000001:0.0:1788572598.646079:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.646083:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a680 x1875440809981184/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.646092:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.646095:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440809981184/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.646101:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.646103:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.649942:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.649958:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.649975:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.649979:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.649988:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.649995:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.650003:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.650008:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.650013:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.650017:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 432 into portal 4 MB=0x6a9b41746b500 00000400:00000200:0.0:1788572598.650022:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.650029:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 432/432 into md 0x52a909 [1] + 224 00000800:00000001:0.0:1788572598.650036:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.650044:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.650047:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.650051:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.650067:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.650075:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a680 x1875440809981184/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.650087:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=432 offset=224 replen=432 req@ffff9c371ed6a680 x1875440809981184/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.650111:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.650119:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.650122:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.651113:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.651119:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5c00. 00000400:00000200:0.0:1788572598.651123:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.651128:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.651132:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf14c8 00000400:00000010:0.0:1788572598.651134:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf14c8. 00000100:00000001:0.0:1788572598.651137:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.651142:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a300 x1875440809981440/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572614 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.651152:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.651160:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6a300 x1875440809981440/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572614 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.651167:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.651169:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.654434:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.654450:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.654454:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.654469:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.654478:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.654484:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.654491:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.654495:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.654499:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.654502:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746b600 00000400:00000200:0.0:1788572598.654506:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.654511:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a911 [1] + 224 00000800:00000001:0.0:1788572598.654526:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.654534:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.654538:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.654542:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.654548:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.654568:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6a300 x1875440809981440/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.654578:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=624 req@ffff9c371ed6a300 x1875440809981440/t0(0) o35->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/624 e 0 to 0 dl 1788572614 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.654799:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.654809:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.654812:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.662415:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.662420:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5e00. 00000400:00000200:0.0:1788572598.662424:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.662430:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.662436:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1770 00000400:00000010:0.0:1788572598.662444:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1770. 00000100:00000001:0.0:1788572598.662449:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.662453:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809981568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.662460:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.662462:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68380 x1875440809981568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.662468:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.662470:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.669248:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.669264:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.669268:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.669270:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.669280:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.669287:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.669297:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.669302:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.669307:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.669312:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746b680 00000400:00000200:0.0:1788572598.669317:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.669323:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a919 [1] + 224 00000800:00000001:0.0:1788572598.669330:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.669338:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.669341:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.669345:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.669352:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.669361:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68380 x1875440809981568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.669372:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371ed68380 x1875440809981568/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.669389:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.669396:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.669411:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.674138:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.674143:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5100. 00000400:00000200:0.0:1788572598.674148:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.674153:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.674158:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1aa0 00000400:00000010:0.0:1788572598.674160:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1aa0. 00000100:00000001:0.0:1788572598.674164:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.674168:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68380 x1875440809981824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.674177:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.674180:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68380 x1875440809981824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.674187:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.674188:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.678358:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.678373:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.678377:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.678378:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.678385:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.678390:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.678397:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.678402:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.678408:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.678412:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b41746b780 00000400:00000200:0.0:1788572598.678417:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.678423:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a921 [32] + 224 00000800:00000001:0.0:1788572598.678429:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.678436:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.678439:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.678444:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.678451:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.678459:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68380 x1875440809981824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.678483:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66584 req@ffff9c371ed68380 x1875440809981824/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.678507:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.678516:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.678520:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.680486:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.680491:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498c00. 00000400:00000200:0.0:1788572598.680495:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.680501:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.680506:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1ee0 00000400:00000010:0.0:1788572598.680508:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1ee0. 00000100:00000001:0.0:1788572598.680511:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.680516:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70000 x1875440809982080/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.680525:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.680528:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec70000 x1875440809982080/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.680535:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.680537:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.684288:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.684303:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.684308:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.684311:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.684320:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.684327:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.684336:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.684340:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.684346:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.684350:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746b880 00000400:00000200:0.0:1788572598.684355:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.684361:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a929 [1] + 224 00000800:00000001:0.0:1788572598.684383:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.684401:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.684405:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.684411:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.684419:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.684428:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70000 x1875440809982080/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.684440:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=456 offset=224 replen=512 req@ffff9c371ec70000 x1875440809982080/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.684470:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.684479:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.684483:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.685406:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.685411:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498700. 00000400:00000200:0.0:1788572598.685421:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.685429:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.685433:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1f68 00000400:00000010:0.0:1788572598.685435:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1f68. 00000100:00000001:0.0:1788572598.685438:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.685443:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809982208/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.685452:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.685455:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ec71f80 x1875440809982208/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.685461:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.685463:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.688859:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.688874:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.688878:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.688881:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.688889:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.688897:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.688905:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.688958:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.688963:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.688968:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 10 MB=0x6a9b41746b900 00000400:00000200:0.0:1788572598.688973:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.688979:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a931 [1] + 224 00000800:00000001:0.0:1788572598.688987:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.688995:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.688998:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.689003:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.689011:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.689019:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809982208/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.689031:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=736 req@ffff9c371ec71f80 x1875440809982208/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.689052:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.689059:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.689075:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.690712:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.690717:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498200. 00000400:00000200:0.0:1788572598.690721:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.690725:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.690729:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1880 00000400:00000010:0.0:1788572598.690731:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1880. 00000100:00000001:0.0:1788572598.690734:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.690738:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71f80 x1875440809982336/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.690746:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.690749:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440809982336/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.690755:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.690756:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.694413:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.694440:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.694444:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.694446:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.694454:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.694461:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.694470:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.694474:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.694478:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.694482:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 432 into portal 4 MB=0x6a9b41746b980 00000400:00000200:0.0:1788572598.694487:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.694493:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 432/432 into md 0x52a939 [1] + 224 00000800:00000001:0.0:1788572598.694499:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.694519:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.694522:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.694526:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.694533:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.694541:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71f80 x1875440809982336/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.694551:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=432 offset=224 replen=432 req@ffff9c371ec71f80 x1875440809982336/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.694575:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.694584:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.694587:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572598.694869:0:266365:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572598.694872:0:266365:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694874:0:266365:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572598.694877:0:266365:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572598.694879:0:266365:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572598.694881:0:266365:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572598.694882:0:266365:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572598.694883:0:266365:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572598.694884:0:266365:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572598.694885:0:266365:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694887:0:266365:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694888:0:266365:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572598.694893:0:266365:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572598.694894:0:266365:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572598.694895:0:266365:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572598.694897:0:266365:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572598.694898:0:266365:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572598.694899:0:266365:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572598.694900:0:266365:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572598.694902:0:266365:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572598.694902:0:266365:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694903:0:266365:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694904:0:266365:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572598.694905:0:266365:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572598.694915:0:266365:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572598.694917:0:266365:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572598.694919:0:266365:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572598.694920:0:266365:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572598.694921:0:266365:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572598.694922:0:266365:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572598.694925:0:266365:0:(cl_object.c:917:cl_env_put()) 1@ffff9c373184d1b0 00020000:00000001:0.0:1788572598.694927:0:266365:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572598.694929:0:266365:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572598.694931:0:266365:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572598.694933:0:266365:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdfd:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572598.694935:0:266365:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572598.694936:0:266365:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572598.694936:0:266365:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572598.694938:0:266365:0:(lcommon_cl.c:108:cl_setattr_ost()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572598.694941:0:266365:0:(llite_lib.c:2656:ll_setattr_raw()) Process leaving 00000080:00000010:0.0:1788572598.694944:0:266365:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3731385600. 00000020:00000001:0.0:1788572598.694950:0:266365:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d27c30 00000080:00200008:0.0:1788572598.694958:0:266365:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f11:[0x200002b17:0xdfd:0x0](ffff9c37020e4b08) current size 0, valid attrs 100201f0, mode 0, uid -1055292862, gid -1, new size 18446744072466397719, atime 1788572598.000000000, mtime 1788572598.000000000, ctime 1788572598.000000000, rc 0 00000080:00000001:0.0:1788572598.694962:0:266365:0:(llite_lib.c:2688:ll_setattr_raw()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572598.694976:0:266365:0:(lov_object.c:2398:lov_read_and_clear_async_rc()) Process entered 00020000:00000002:0.0:1788572598.694978:0:266365:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cbaba0) owner 0000000000000000/ffff9c37012a0000 00020000:00000002:0.0:1788572598.694982:0:266365:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cbaba0) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572598.694984:0:266365:0:(lov_object.c:2449:lov_read_and_clear_async_rc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572598.694989:0:266365:0:(file.c:475:ll_file_release()) Process entered 00000020:00000001:0.0:1788572598.694991:0:266365:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1d27df0 00000080:00200008:0.0:1788572598.694996:0:266365:0:(file.c:479:ll_file_release()) START file f11:[0x200002b17:0xdfd:0x0](ffff9c37020e4b08), flags 104001 00000080:00000001:0.0:1788572598.694998:0:266365:0:(pcc.c:2472:pcc_file_release()) Process entered 00000080:00000001:0.0:1788572598.695000:0:266365:0:(pcc.c:2500:pcc_file_release()) Process leaving 00020000:00000001:0.0:1788572598.695001:0:266365:0:(lov_object.c:2398:lov_read_and_clear_async_rc()) Process entered 00020000:00000002:0.0:1788572598.695002:0:266365:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cbaba0) owner 0000000000000000/ffff9c37012a0000 00020000:00000002:0.0:1788572598.695004:0:266365:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cbaba0) owner 0000000000000000/ffff9c37012a0000 00020000:00000001:0.0:1788572598.695005:0:266365:0:(lov_object.c:2449:lov_read_and_clear_async_rc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572598.695006:0:266365:0:(file.c:341:ll_md_close()) Process entered 00800000:00000001:0.0:1788572598.695011:0:266365:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572598.695013:0:266365:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572598.695015:0:266365:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572598.695016:0:266365:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572598.695017:0:266365:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572598.695019:0:266365:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfd:0x0] 00800000:00000001:0.0:1788572598.695020:0:266365:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572598.695024:0:266365:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572598.695026:0:266365:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572598.695030:0:266365:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3704ff6900 count: 2 00010000:00000001:0.0:1788572598.695032:0:266365:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572598.695036:0:266365:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572598.695038:0:266365:0:(lmv_obd.c:4142:lmv_lock_match()) Lock match for [0x200002b17:0xdfd:0x0]: 0 00000080:00000001:0.0:1788572598.695039:0:266365:0:(file.c:297:ll_md_real_close()) Process entered 00000080:00000001:0.0:1788572598.695041:0:266365:0:(file.c:150:ll_close_inode_openhandle()) Process entered 00000080:00000010:0.0:1788572598.695043:0:266365:0:(file.c:158:ll_close_inode_openhandle()) kmalloced '(op_data)': 392 at ffff9c3711c20600. 00000080:00000001:0.0:1788572598.695044:0:266365:0:(file.c:90:ll_prepare_close()) Process entered 00000080:00000001:0.0:1788572598.695048:0:266365:0:(file.c:128:ll_prepare_close()) Process leaving 00800000:00000001:0.0:1788572598.695051:0:266365:0:(lmv_obd.c:1670:lmv_close()) Process entered 00800000:00000001:0.0:1788572598.695052:0:266365:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572598.695053:0:266365:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572598.695053:0:266365:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572598.695054:0:266365:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572598.695055:0:266365:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572598.695056:0:266365:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfd:0x0] 00800000:00000001:0.0:1788572598.695057:0:266365:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572598.695058:0:266365:0:(lmv_obd.c:1676:lmv_close()) CLOSE [0x200002b17:0xdfd:0x0] 00000002:00000001:0.0:1788572598.695060:0:266365:0:(mdc_request.c:879:mdc_close()) Process entered 00000002:00000002:0.0:1788572598.695062:0:266365:0:(mdc_request.c:882:mdc_close()) lustre-MDT0000-mdc-ffff9c3711155800: [0x200002b17:0xdfd:0x0] file closed with intent: 0 00000100:00000010:0.0:1788572598.695065:0:266365:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572598.695071:0:266365:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=6 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572598.695075:0:266365:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572598.695077:0:266365:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572598.695080:0:266365:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572598.695085:0:266365:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3711c21c00. 00000100:00000001:0.0:1788572598.695090:0:266365:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00100000:0.0:1788572598.695103:0:266365:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff9c371ed68380 x1875440809981824/t25769859806(25769859806) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572683 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788572598.695110:0:266365:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788572598.695113:0:266365:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b4700 x1875440809982592/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00000001:0.0:1788572598.695118:0:266365:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572598.695119:0:266365:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572598.695123:0:266365:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a840. 00000100:00000001:0.0:1788572598.695124:0:266365:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040026688 : -109714669524928 : ffff9c371018a840) 00000100:00000001:0.0:1788572598.695126:0:266365:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572598.695127:0:266365:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788572598.695130:0:266365:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572598.695134:0:266365:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.695135:0:266365:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572598.695137:0:266365:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572598.695141:0:266365:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809982592/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.695147:0:266365:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572598.695149:0:266365:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572598.695153:0:266365:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572598.695155:0:266365:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572598.695158:0:266365:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572598.695161:0:266365:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266365:1875440809982592:192.168.202.148@tcp:35:touch.0 00000100:00000001:0.0:1788572598.695165:0:266365:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572598.695169:0:266365:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572598.695171:0:266365:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572598.695178:0:266365:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572598.695183:0:266365:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ed000. 02000000:00000001:0.0:1788572598.695185:0:266365:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572598.695190:0:266365:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572598.695192:0:266365:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788572598.695196:0:266365:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572598.695202:0:266365:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809982592, portal 10 00000100:00000001:0.0:1788572598.695203:0:266365:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572598.695204:0:266365:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572598.695209:0:266365:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4700 x1875440809982592/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572614 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.695214:0:266365:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572598.695215:0:266365:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572598.695217:0:266365:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572598.695219:0:266365:0:(niobuf.c:94:ptl_send_buf()) Sending 392 bytes to portal 23, xid 1875440809982592, offset 0 00000400:00000200:0.0:1788572598.695222:0:266365:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572598.695226:0:266365:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.695229:0:266365:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572598.695233:0:266365:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572598.695236:0:266365:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572598.695238:0:266365:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572598.695240:0:266365:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572598.695243:0:266365:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540036:540036:256:4294967295] 192.168.202.148@tcp LPNI seq info [540001:540001:8:4294967295] 00000400:00000200:0.0:1788572598.695246:0:266365:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.695252:0:266365:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572598.695255:0:266365:0:(socklnd_cb.c:931:ksocknal_send()) sending 392 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572598.695289:0:266365:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572598.695295:0:266365:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572598.695300:0:266365:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572598.695303:0:266365:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0d00 type 1, nob 488 niov 1 nkiov 1 00000100:00000001:0.0:1788572598.695317:0:266365:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572598.695324:0:266365:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15101 usecs (16 jiffies) 00000400:00000200:0.0:1788572598.695327:0:266365:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572598.695329:0:266365:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307737717 00000100:00000001:0.0:1788572598.695331:0:266365:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307737866 00000100:00000001:0.0:1788572598.695332:0:266365:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307737717 00000100:00000001:0.0:1788572598.695334:0:266365:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307737866 00000100:00000001:0.0:1788572598.695335:0:266365:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572598.695337:0:266365:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.695338:0:266365:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572598.695339:0:266365:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1788572598.695340:0:266365:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a840 going to sleep for 11 seconds 00000100:00000001:0.0:1788572598.695341:0:266365:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572598.695343:0:266365:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572598.697937:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.697950:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.697955:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.697957:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.697965:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.697982:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.697991:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.697995:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.697999:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.698003:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746ba80 00000400:00000200:0.0:1788572598.698009:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.698015:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a941 [1] + 224 00000800:00000001:0.0:1788572598.698030:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.698039:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.698042:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.698046:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.698053:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.698061:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b4700 x1875440809982592/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.698105:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=888 req@ffff9c372d6b4700 x1875440809982592/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572614 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.698125:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.698133:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.698137:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.707882:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.707895:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.707898:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.707900:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.707917:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.707924:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.707936:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.707940:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.707946:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.707949:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746bb00 00000400:00000200:0.0:1788572598.707955:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.707961:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52a951 [1] + 224 00000800:00000001:0.0:1788572598.707968:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.707975:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.707979:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.707983:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.707990:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.707999:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809982720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.708010:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371ec70e00 x1875440809982720/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.708027:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.708035:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.708038:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.713975:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.714002:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.714007:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.714009:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.714015:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.714021:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.714028:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.714031:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.714035:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.714038:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b41746bc00 00000400:00000200:0.0:1788572598.714042:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.714047:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52a959 [32] + 224 00000800:00000001:0.0:1788572598.714052:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.714056:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.714059:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.714062:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.714074:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.714080:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70e00 x1875440809982976/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.714087:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66584 req@ffff9c371ec70e00 x1875440809982976/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.714100:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.714107:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.714110:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.719223:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.719237:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.719242:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.719244:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.719252:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.719260:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.719269:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.719274:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.719278:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.719286:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746bd00 00000400:00000200:0.0:1788572598.719302:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.719306:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a961 [1] + 224 00000800:00000001:0.0:1788572598.719312:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.719318:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.719320:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.719323:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.719328:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.719334:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec73480 x1875440809983232/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.719341:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=456 offset=224 replen=512 req@ffff9c371ec73480 x1875440809983232/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.719353:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.719359:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.719361:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.722419:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.722433:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.722725:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.722732:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.722741:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.722749:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.722757:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.722761:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.722765:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.722769:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 10 MB=0x6a9b41746bd80 00000400:00000200:0.0:1788572598.722774:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.722780:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 400/400 into md 0x52a969 [1] + 224 00000800:00000001:0.0:1788572598.722786:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.722794:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.722797:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.722801:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.722806:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.722813:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72a00 x1875440809983360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.722837:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=736 req@ffff9c371ec72a00 x1875440809983360/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572683 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.722854:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.722862:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.722866:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.726240:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.726258:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.726260:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.726262:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.726267:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.726272:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.726277:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.726280:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.726284:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.726287:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 432 into portal 4 MB=0x6a9b41746be00 00000400:00000200:0.0:1788572598.726292:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.726297:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 432/432 into md 0x52a971 [1] + 224 00000800:00000001:0.0:1788572598.726307:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.726314:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.726317:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.726321:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.726326:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.726333:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72a00 x1875440809983488/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.726342:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=432 offset=224 replen=432 req@ffff9c371ec72a00 x1875440809983488/t0(0) o2->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.726357:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.726364:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.726368:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.730121:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.730148:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.730152:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.730153:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.730159:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.730164:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.730170:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.730174:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.730178:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.730181:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746bf00 00000400:00000200:0.0:1788572598.730184:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.730188:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52a979 [1] + 224 00000800:00000001:0.0:1788572598.730193:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.730197:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.730199:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.730202:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.730206:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.730212:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec70380 x1875440809983744/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572598.730219:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=888 req@ffff9c371ec70380 x1875440809983744/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572614 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572598.730234:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.730251:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.730254:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572598.737706:0:266368:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572598.737711:0:266368:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572598.737714:0:266368:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572598.737723:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737726:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737731:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737733:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737736:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737738:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737741:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737748:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737752:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737754:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737757:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737758:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737761:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737763:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737766:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737769:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737772:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737775:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737778:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737779:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737782:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737784:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737788:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737789:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737793:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737796:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737802:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737805:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737811:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737815:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737820:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737822:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737824:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737828:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737833:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737837:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737843:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737848:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737854:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737858:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737861:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737869:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737874:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737876:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737878:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737886:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737891:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737893:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737896:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737898:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737901:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737903:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737918:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737919:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737923:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737928:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737933:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737936:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737942:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737946:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737950:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737952:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737955:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737958:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737962:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737965:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737968:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737970:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737975:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737979:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737984:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737988:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.737994:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.737997:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738000:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738010:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738017:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738022:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738028:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738033:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738037:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738040:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738044:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738047:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738051:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738053:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738058:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738061:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738066:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738069:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738074:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738077:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738082:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738085:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738088:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738091:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738096:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738099:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738106:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738110:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738117:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738122:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738127:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738130:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738136:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738140:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738146:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738149:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738152:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738161:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738168:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738171:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738178:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738181:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738186:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738189:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738193:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738196:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738200:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738204:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738209:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738214:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738219:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738223:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738228:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738232:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738237:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738241:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738248:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738252:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738257:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738261:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738267:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738271:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738277:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738280:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738285:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738288:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738294:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738298:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738302:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738303:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738306:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738315:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738320:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738323:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738329:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738333:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738338:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738341:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738346:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738349:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738355:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738359:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738365:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738368:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738372:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738376:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738381:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738384:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738389:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738393:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738398:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738402:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738407:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738412:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738416:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738419:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738424:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738428:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738433:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738436:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738442:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738446:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738451:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738453:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738455:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738465:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738471:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738474:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738480:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738484:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738488:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738492:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738497:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738501:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738505:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738508:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738512:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738516:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738521:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738525:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738531:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738535:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738539:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738541:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738546:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738549:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738554:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738557:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738562:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738565:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738569:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738573:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738577:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738580:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738585:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738589:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738593:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738596:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738599:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738607:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738610:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738613:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738617:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738620:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738622:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738624:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738630:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738633:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738668:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738672:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738679:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738681:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738685:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738689:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738697:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738700:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738704:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738707:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738712:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738715:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738719:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738723:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738727:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738729:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738734:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738736:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738741:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738744:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738748:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738751:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738756:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738759:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738763:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738771:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738776:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738778:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738784:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738787:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738791:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738794:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738799:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738801:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738806:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738809:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738815:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738818:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738822:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738823:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738826:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738829:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738832:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738834:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738836:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738839:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738843:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738847:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738851:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738854:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738859:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738863:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738868:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738871:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738878:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738881:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738886:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738889:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738894:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738919:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738925:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738928:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738931:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738936:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738941:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738943:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738945:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738948:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738953:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738956:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738961:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738963:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738966:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738969:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738972:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738975:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738981:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738984:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738989:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738991:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.738995:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.738999:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739004:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739007:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739011:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739016:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739021:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739024:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739028:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739031:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739034:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739037:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739041:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739051:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739055:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739058:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739062:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739065:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739071:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739074:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739078:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739082:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739085:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739087:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739089:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739092:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739096:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739099:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739104:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739107:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739111:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739112:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739115:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739118:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739124:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739128:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739134:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739138:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739142:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739146:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739149:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739152:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739156:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739159:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739164:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739167:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739171:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739180:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739184:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739188:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739191:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739194:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739201:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739206:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739211:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739216:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739222:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739227:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739232:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739236:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739241:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739245:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739250:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739254:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739258:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739259:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739262:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739264:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739266:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739268:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739270:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739272:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739274:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739275:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739278:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739279:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739281:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739283:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739285:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739286:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739288:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739295:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739299:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739301:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739305:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739307:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739312:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739315:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739319:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739322:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739326:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739329:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739334:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739337:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739340:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739343:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739349:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739351:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739355:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739358:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739363:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739365:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739370:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739374:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739378:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739381:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739385:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739388:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739393:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739396:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739400:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739403:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739407:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739408:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739410:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739415:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739418:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739420:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739423:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739424:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739427:0:266368:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572598.739429:0:266368:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.739430:0:266368:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=211 : 211 : d3) 00010000:00000001:0.0:1788572598.739433:0:266368:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572598.739435:0:266368:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:0.0:1788572598.739448:0:266368:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 211/211 locks to regular blp list, 1 blwis in pool 00000800:00000001:0.0:1788572598.742960:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.742973:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.742976:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.742978:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.742985:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.742992:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.743000:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.743003:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.743007:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.743009:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41746bf80 00000400:00000200:0.0:1788572598.743013:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.743017:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a981 [1] + 224 00000800:00000001:0.0:1788572598.743022:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.743027:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.743029:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.743033:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.743038:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.743043:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec72d80 x1875440809983872/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 344/224 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.743050:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ec72d80 x1875440809983872/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 344/224 e 0 to 0 dl 1788572614 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.743064:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.743077:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.743080:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572598.762762:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572598.762767:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572598.762773:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.762778:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572598.762783:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3440 00000400:00000010:0.0:1788572598.762787:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3440. 00000100:00000001:0.0:1788572598.762791:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572598.762796:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7480 x1875440809984000/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 2000/224 e 0 to 0 dl 1788572614 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.762807:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572598.762811:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c7480 x1875440809984000/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 2000/224 e 0 to 0 dl 1788572614 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.762818:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572598.762821:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572598.780982:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.780997:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572598.781002:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.781004:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.781014:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.781022:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572598.781031:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.781036:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.781042:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.781046:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41746c000 00000400:00000200:0.0:1788572598.781051:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572598.781058:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52a989 [1] + 224 00000800:00000001:0.0:1788572598.781065:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.781073:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572598.781076:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572598.781081:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572598.781089:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572598.781111:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c7480 x1875440809984000/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 2000/224 e 0 to 0 dl 1788572614 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572598.781122:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371f1c7480 x1875440809984000/t0(0) o103->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 2000/224 e 0 to 0 dl 1788572614 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572598.781139:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572598.781148:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572598.781151:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572598.785273:0:266368:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572598.785278:0:266368:0:(ldlm_lockd.c:2264:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572598.785280:0:266368:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=211 : 211 : d3) 00000100:00000040:0.0:1788572598.889752:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307737892, at 4307737896 00000100:00000010:0.0:1788572598.889769:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3731cf6a80. 00000080:00000001:0.0:1788572599.449167:0:266396:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.449171:0:266396:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.449173:0:266396:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572599.449177:0:266396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fba8 00000080:00200000:0.0:1788572599.449182:0:266396:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572599.449188:0:266396:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20a00. 00800000:00000002:0.0:1788572599.449194:0:266396:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572599.449199:0:266396:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.449203:0:266396:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.449205:0:266396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9fac8 00800000:00000002:0.0:1788572599.449208:0:266396:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.449211:0:266396:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.449214:0:266396:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.449217:0:266396:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.449219:0:266396:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.449221:0:266396:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.449223:0:266396:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.449225:0:266396:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.449228:0:266396:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.449231:0:266396:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572599.449235:0:266396:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.449237:0:266396:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1e9f998 00000002:00010000:0.0:1788572599.449245:0:266396:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572599.449249:0:266396:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572599.449251:0:266396:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.449253:0:266396:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.449259:0:266396:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.449261:0:266396:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.449263:0:266396:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.449265:0:266396:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.449267:0:266396:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572599.449271:0:266396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572599.449275:0:266396:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=10 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572599.449280:0:266396:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.449284:0:266396:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.449288:0:266396:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.449290:0:266396:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.449293:0:266396:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.449298:0:266396:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.449303:0:266396:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572599.449310:0:266396:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.449314:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.449316:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.449318:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.449319:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.449321:0:266396:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.449325:0:266396:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00010000:00000001:0.0:1788572599.449329:0:266396:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.449331:0:266396:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572599.449335:0:266396:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46800. 00010000:00000010:0.0:1788572599.449339:0:266396:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676280. 00010000:00000001:0.0:1788572599.449344:0:266396:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.449348:0:266396:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30e00. 00000020:00000001:0.0:1788572599.449351:0:266396:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.449355:0:266396:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30e00 with handle 0x1441586093168923 to hash 00000020:00000001:0.0:1788572599.449357:0:266396:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.449358:0:266396:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.449367:0:266396:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.449370:0:266396:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.449373:0:266396:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.449378:0:266396:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 266396 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.449387:0:266396:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 266396 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.449398:0:266396:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 266396 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.449404:0:266396:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.449406:0:266396:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.449412:0:266396:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018ac00. 00000100:00000001:0.0:1788572599.449414:0:266396:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040027648 : -109714669523968 : ffff9c371018ac00) 00000100:00000001:0.0:1788572599.449417:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.449419:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000001:0.0:1788572599.449424:0:266396:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.449427:0:266396:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.449430:0:266396:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.449432:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.449441:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809984128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.449450:0:266396:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.449453:0:266396:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.449456:0:266396:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.449459:0:266396:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.449462:0:266396:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.449466:0:266396:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266396:1875440809984128:192.168.202.148@tcp:101:mkdir.0 00000100:00000001:0.0:1788572599.449470:0:266396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.449475:0:266396:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.449478:0:266396:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.449485:0:266396:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.449490:0:266396:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec000. 02000000:00000001:0.0:1788572599.449492:0:266396:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.449500:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.449505:0:266396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c87e8. 00000400:00000010:0.0:1788572599.449512:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572599.449519:0:266396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809984128, portal 10 00000100:00000001:0.0:1788572599.449521:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.449522:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572599.449529:0:266396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b4000 x1875440809984128/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.449537:0:266396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.449540:0:266396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.449543:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572599.449547:0:266396:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1875440809984128, offset 0 00000400:00000200:0.0:1788572599.449555:0:266396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.449562:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.449570:0:266396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.449575:0:266396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.449580:0:266396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.449584:0:266396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.449587:0:266396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.449591:0:266396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540045:540045:256:4294967295] 192.168.202.148@tcp LPNI seq info [540010:540010:8:4294967295] 00000400:00000200:0.0:1788572599.449596:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.449606:0:266396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.449614:0:266396:0:(socklnd_cb.c:931:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.449620:0:266396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572599.449627:0:266396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.449633:0:266396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.449662:0:266396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.449677:0:266396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.449689:0:266396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15080 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.449693:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.449696:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738469 00000100:00000010:0.0:1788572599.449705:0:266396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116269c0. 00000100:00000001:0.0:1788572599.449711:0:266396:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572599.449717:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738469 00000100:00000010:0.0:1788572599.449721:0:266396:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626780. 00000100:00000001:0.0:1788572599.449725:0:266396:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572599.449728:0:266396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.449731:0:266396:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.449734:0:266396:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.449736:0:266396:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.449739:0:266396:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018ac00 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.449742:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.449745:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.452823:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.452827:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.452830:0:266397:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572599.452834:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafbc0 00000080:00200000:0.0:1788572599.452839:0:266397:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572599.452847:0:266397:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20600. 00800000:00000002:0.0:1788572599.452855:0:266397:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 3 00000080:00000001:0.0:1788572599.452861:0:266397:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.452866:0:266397:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.452869:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafae0 00800000:00000002:0.0:1788572599.452872:0:266397:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.452889:0:266397:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.452893:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.452897:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.452900:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.452905:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.452909:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.452913:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.452917:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.452923:0:266397:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572599.452937:0:266397:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.452940:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf9b0 00000002:00010000:0.0:1788572599.452944:0:266397:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572599.452950:0:266397:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572599.452954:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.452958:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.452965:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46800 count: 3 00010000:00000001:0.0:1788572599.452972:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.452979:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.452982:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.452987:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 4/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1788572599.452998:0:266397:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572599.453003:0:266397:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.453006:0:266397:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 4/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000001:0.0:1788572599.453018:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572599.453022:0:266397:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572599.453026:0:266397:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.453029:0:266397:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.453034:0:266397:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.453036:0:266397:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572599.453040:0:266397:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572599.453046:0:266397:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 3 00000080:00000010:0.0:1788572599.453052:0:266397:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20600. 00000080:00000001:0.0:1788572599.453061:0:266397:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572599.453065:0:266397:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.453069:0:266397:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572599.453075:0:266397:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093168923 00800000:00000001:0.0:1788572599.453082:0:266397:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.453086:0:266397:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.453097:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.453101:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.453105:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=7 00000020:00000001:0.0:1788572599.453110:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.453115:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.453127:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.453130:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.453133:0:266397:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.453136:0:266397:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.453142:0:266397:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168923 from it ffffb95bc1eafbf8 00010000:00000001:0.0:1788572599.453146:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.453149:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.453153:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=4 00000020:00000001:0.0:1788572599.453158:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.453162:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.453166:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.453172:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.453185:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.453187:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.453191:0:266397:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.453199:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.453201:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.453203:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572599.453206:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572599.453208:0:266397:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.453212:0:266397:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572599.453218:0:266397:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572599.453227:0:266397:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f291580. 00000020:00000001:0.0:1788572599.453233:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafb28 00000080:00200000:0.0:1788572599.453237:0:266397:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d801b.sanity, dir=[0x200000007:0x1:0x0](ffff9c372dcc6c08), flags=25 00000080:00000001:0.0:1788572599.453249:0:266397:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572599.453253:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf9d8 00000080:00200000:0.0:1788572599.453256:0:266397:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d801b.sanity, dir=[0x200000007:0x1:0x0](ffff9c372dcc6c08), intent=0 00000080:00000010:0.0:1788572599.453267:0:266397:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20600. 00800000:00000002:0.0:1788572599.453273:0:266397:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 3 00000080:00000001:0.0:1788572599.453278:0:266397:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.453281:0:266397:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.453283:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf8f8 00800000:00000002:0.0:1788572599.453286:0:266397:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'lookup' for [0x0:0x0:0x0] 'd801b.sanity' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.453292:0:266397:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.453294:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.453298:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.453301:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.453305:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.453308:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.453312:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.453318:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.453323:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf890 00800000:00000002:0.0:1788572599.453326:0:266397:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x0:0x0:0x0], name='d801b.sanity' -> mds #0 00000002:00000001:0.0:1788572599.453329:0:266397:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.453332:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf7c8 00000002:00010000:0.0:1788572599.453335:0:266397:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d801b.sanity,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00000002:00000001:0.0:1788572599.453340:0:266397:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.453344:0:266397:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572599.453351:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572599.453357:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572599.453363:0:266397:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.453369:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.453374:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.453379:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.453383:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.453388:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.453396:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572599.453404:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.453410:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.453421:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.453426:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.453429:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.453436:0:266397:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.453446:0:266397:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572599.453451:0:266397:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.453455:0:266397:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.453460:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46800 count: 2 00010000:00000001:0.0:1788572599.453467:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.453471:0:266397:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.453477:0:266397:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31c00. 00000020:00000001:0.0:1788572599.453480:0:266397:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.453484:0:266397:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31c00 with handle 0x144158609316892a to hash 00000020:00000001:0.0:1788572599.453487:0:266397:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.453490:0:266397:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.453494:0:266397:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.453499:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.453501:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.453506:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.453515:0:266397:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.453522:0:266397:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.453530:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.453534:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.453539:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.453543:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040025536 : -109714669526080 : ffff9c371018a3c0) 00000100:00000001:0.0:1788572599.453546:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.453549:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572599.453553:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.453564:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.453570:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.453574:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.453584:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.453599:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.453602:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.453606:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.453608:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.453613:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.453619:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809984256:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.453627:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.454082:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.454086:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.454089:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.454094:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572599.454097:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.454103:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.454108:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788572599.454114:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572599.454120:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809984256, portal 10 00000100:00000001:0.0:1788572599.454122:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.454123:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572599.454129:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.454137:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.454141:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.454144:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572599.454147:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 584 bytes to portal 12, xid 1875440809984256, offset 0 00000400:00000200:0.0:1788572599.454154:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.454162:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.454168:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.454174:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.454184:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.454188:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.454190:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.454193:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540046:540046:256:4294967295] 192.168.202.148@tcp LPNI seq info [540011:540011:8:4294967295] 00000400:00000200:0.0:1788572599.454196:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.454204:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.454209:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 584 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.454214:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788572599.454219:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.454227:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.454231:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0e00 type 1, nob 680 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.454253:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.454257:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15076 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.454261:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.454264:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738476 00000100:00000001:0.0:1788572599.454272:0:266397:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307738469 00000100:00000001:0.0:1788572599.454275:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738476 00000100:00000001:0.0:1788572599.454279:0:266397:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307738469 00000100:00000001:0.0:1788572599.454282:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.454283:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.454285:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.454287:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.454289:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a3c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.454291:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.454293:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.457895:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.457902:0:266397:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.457903:0:266397:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.457906:0:266397:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788572599.457911:0:266397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31dd0 00000400:00000010:0.0:1788572599.457912:0:266397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31dd0. 00000100:00000001:0.0:1788572599.457921:0:266397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.457925:0:266397:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.457934:0:266397:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.457940:0:266397:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.457942:0:266397:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.457944:0:266397:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.457946:0:266397:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.457948:0:266397:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.457951:0:266397:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.457953:0:266397:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.457961:0:266397:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.457965:0:266397:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=45 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572599.457968:0:266397:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572599.457969:0:266397:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.457971:0:266397:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572599.457974:0:266397:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.457977:0:266397:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.457978:0:266397:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572599.457980:0:266397:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.457982:0:266397:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.457987:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/584 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.457993:0:266397:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.457996:0:266397:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.457997:0:266397:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.458001:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/584 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.458005:0:266397:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809984256:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.458009:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.458012:0:266397:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.458021:0:266397:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.458023:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.458027:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/584 e 0 to 0 dl 1788572684 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.458033:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.458036:0:266397:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.458038:0:266397:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.458040:0:266397:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458042:0:266397:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.458046:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458047:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458049:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=4 00000020:00000001:0.0:1788572599.458051:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458054:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000040:0.0:1788572599.458059:0:266397:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372db31c00, remote cookie: 0xd9894672d99a2f56, flags: 0x1001 00010000:00010000:0.0:1788572599.458063:0:266397:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572599.458068:0:266397:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdf9:0x0].0x0, instead of [0x200000007:0x1:0x0].0x0 00010000:00000001:0.0:1788572599.458070:0:266397:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572599.458074:0:266397:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46d00. 00010000:00000010:0.0:1788572599.458076:0:266397:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676380. 00010000:00000001:0.0:1788572599.458081:0:266397:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.458084:0:266397:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.458092:0:266397:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458098:0:266397:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572599.458102:0:266397:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572599.458108:0:266397:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572599.458109:0:266397:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572599.458111:0:266397:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572599.458114:0:266397:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdf9:0x0].0x0 (ffff9c3713c46d00) refcount = 1 00010000:00010000:0.0:1788572599.458118:0:266397:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458123:0:266397:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572599.458125:0:266397:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572599.458126:0:266397:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.458128:0:266397:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572599.458130:0:266397:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.458132:0:266397:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458135:0:266397:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572599.458136:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458137:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458137:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458138:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458139:0:266397:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.458142:0:266397:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572599.458143:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458144:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458145:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458146:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458148:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458149:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458150:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458150:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=4 00000020:00000001:0.0:1788572599.458151:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458152:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458154:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.458155:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.458160:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458164:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458165:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458166:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458166:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458167:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=5 00000020:00000001:0.0:1788572599.458168:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458169:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458170:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.458172:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458175:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458175:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.458177:0:266397:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458180:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.458180:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458181:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458182:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458183:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572599.458186:0:266397:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572599.458191:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458192:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458193:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458194:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458195:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458197:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458199:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.458200:0:266397:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.458202:0:266397:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.458204:0:266397:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572599.458205:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458206:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458207:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458208:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458209:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00000002:00010000:0.0:1788572599.458211:0:266397:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458214:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458215:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458216:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.458217:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458218:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458219:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458221:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458222:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000040:0.0:1788572599.458226:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46d00 count: 2 00010000:00000001:0.0:1788572599.458227:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572599.458230:0:266397:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3577 (0 0) 00010000:00000001:0.0:1788572599.458232:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458233:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.458234:0:266397:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572599.458236:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf750 00000002:00002000:0.0:1788572599.458238:0:266397:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=d801b.sanity intent=lookup status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572599.458240:0:266397:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.458243:0:266397:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.458245:0:266397:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572599.458247:0:266397:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 0 00000080:00000001:0.0:1788572599.458250:0:266397:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.458252:0:266397:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000001:0.0:1788572599.458261:0:266397:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572599.458263:0:266397:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1eafac8 it_disposition b 00000080:00000001:0.0:1788572599.458265:0:266397:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572599.458268:0:266397:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572599.458270:0:266397:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572599.458271:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329145 : 144115373145329145 : 200002b17000df9) 00000080:00000001:0.0:1788572599.458273:0:266397:0:(namei.c:113:ll_iget()) Process entered 00000080:00000001:0.0:1788572599.458277:0:266397:0:(llite_lib.c:1957:ll_update_lsm_md()) Process entered 00000080:00000002:0.0:1788572599.458279:0:266397:0:(llite_lib.c:1961:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200002b17:0xdf9:0x0] 00000080:00000001:0.0:1788572599.458280:0:266397:0:(llite_lib.c:1978:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.458282:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329145 : 144115373145329145 : 200002b17000df9) 00000080:00000001:0.0:1788572599.458284:0:266397:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572599.458287:0:266397:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200002b17:0xdf9:0x0], updating i_size 4096 00000080:00200000:0.0:1788572599.458288:0:266397:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdf9:0x0](ffff9c3708704b08): rc = 0 00000080:00000001:0.0:1788572599.458290:0:266397:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911552264 : -109714797999352 : ffff9c3708704b08) 00010000:00000001:0.0:1788572599.458291:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458292:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458293:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458294:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458296:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458297:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458298:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.458299:0:266397:0:(llite_lib.c:1912:ll_update_default_lsm_md()) Process entered 00000080:00000001:0.0:1788572599.458301:0:266397:0:(llite_lib.c:1925:ll_update_default_lsm_md()) Process leaving 00000080:00000001:0.0:1788572599.458302:0:266397:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572599.458303:0:266397:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572599.458304:0:266397:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.458306:0:266397:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdf9:0x0](ffff9c3708704b08) for lock 0x144158609316892a 00800000:00000001:0.0:1788572599.458307:0:266397:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.458308:0:266397:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.458309:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458309:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458310:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458311:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458312:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458318:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458319:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.458319:0:266397:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.458321:0:266397:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572599.458342:0:266397:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37034e2510 inode ffff9c3708704b08 refc 3 flags 0x20000e 00000080:00000001:0.0:1788572599.458346:0:266397:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.458348:0:266397:0:(llite_lib.c:3161:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.458351:0:266397:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572599.458354:0:266397:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdf9:0x0](ffff9c3708704b08) 00000080:00010000:0.0:1788572599.458358:0:266397:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316892a from it ffffb95bc1eafac8 00010000:00000001:0.0:1788572599.458361:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.458364:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.458366:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=3 00000020:00000001:0.0:1788572599.458370:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458376:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788572599.458380:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.458386:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.458397:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458400:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458404:0:266397:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.458408:0:266397:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.458412:0:266397:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31c00/0x144158609316892a lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f56 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.458427:0:266397:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.458429:0:266397:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.458432:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.458435:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458437:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458439:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458441:0:266397:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.458443:0:266397:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458450:0:266397:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458453:0:266397:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458457:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.458459:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458459:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.458461:0:266397:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572599.458464:0:266397:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20600. 00000100:00000001:0.0:1788572599.458469:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.458474:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809984256/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 584/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.458483:0:266397:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.458487:0:266397:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.458490:0:266397:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572599.458493:0:266397:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.458496:0:266397:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.458497:0:266397:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.458498:0:266397:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.458500:0:266397:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788572599.458502:0:266397:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.458504:0:266397:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.458505:0:266397:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572599.458509:0:266397:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.458512:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.458518:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.458521:0:266397:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.458528:0:266397:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572599.458537:0:266397:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f2914c0. 00000020:00000001:0.0:1788572599.458541:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafb28 00000080:00200000:0.0:1788572599.458545:0:266397:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d1, dir=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), flags=25 00000080:00000001:0.0:1788572599.458552:0:266397:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572599.458555:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf9d8 00000080:00200000:0.0:1788572599.458559:0:266397:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d1, dir=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), intent=0 00000080:00000010:0.0:1788572599.458568:0:266397:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20600. 00000080:00000001:0.0:1788572599.458576:0:266397:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.458580:0:266397:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.458585:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf8f8 00800000:00000002:0.0:1788572599.458597:0:266397:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'lookup' for [0x0:0x0:0x0] 'd1' on [0x200002b17:0xdf9:0x0] 00800000:00000001:0.0:1788572599.458603:0:266397:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.458608:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.458612:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.458616:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.458618:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.458622:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.458627:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdf9:0x0] 00800000:00000001:0.0:1788572599.458631:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.458656:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf890 00800000:00000002:0.0:1788572599.458661:0:266397:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xdf9:0x0], fid2=[0x0:0x0:0x0], name='d1' -> mds #0 00000002:00000001:0.0:1788572599.458667:0:266397:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.458671:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf7c8 00000002:00010000:0.0:1788572599.458676:0:266397:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d1,[0x0:0x0:0x0]) in obj [0x200002b17:0xdf9:0x0], intent: lookup flags 0 00000002:00000001:0.0:1788572599.458686:0:266397:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.458689:0:266397:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572599.458694:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572599.458699:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572599.458705:0:266397:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.458710:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.458713:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.458716:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.458719:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.458723:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.458726:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.458729:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.458735:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572599.458741:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458745:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.458748:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458752:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.458755:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.458758:0:266397:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.458764:0:266397:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572599.458767:0:266397:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.458770:0:266397:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.458779:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46d00 count: 3 00010000:00000001:0.0:1788572599.458783:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.458790:0:266397:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.458796:0:266397:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30c00. 00000020:00000001:0.0:1788572599.458802:0:266397:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.458806:0:266397:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30c00 with handle 0x1441586093168938 to hash 00000020:00000001:0.0:1788572599.458811:0:266397:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.458815:0:266397:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.458820:0:266397:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.458828:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.458832:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.458837:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.458850:0:266397:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.458861:0:266397:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.458870:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.458873:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.458888:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.458893:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040025536 : -109714669526080 : ffff9c371018a3c0) 00000100:00000001:0.0:1788572599.458897:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.458900:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572599.458904:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.458909:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.458914:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.458917:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.458925:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.458944:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.458948:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.458958:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.458962:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.458967:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.458974:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809984512:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.458984:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.458989:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.458993:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.458999:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.459006:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572599.459012:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.459022:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.459028:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788572599.459035:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572599.459042:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809984512, portal 10 00000100:00000001:0.0:1788572599.459046:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.459051:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572599.459063:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.459082:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.459090:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.459096:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788572599.459102:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1875440809984512, offset 0 00000400:00000200:0.0:1788572599.459116:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.459128:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.459135:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.459140:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.459144:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.459150:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.459155:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.459162:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540048:540048:256:4294967295] 192.168.202.148@tcp LPNI seq info [540013:540013:8:4294967295] 00000400:00000200:0.0:1788572599.459174:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.459188:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.459204:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.459213:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572599.459226:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.459241:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.459248:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.459264:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.459272:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15072 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.459280:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.459286:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738481 00000100:00000001:0.0:1788572599.459295:0:266397:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307738476 00000100:00000001:0.0:1788572599.459303:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738481 00000100:00000001:0.0:1788572599.459312:0:266397:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307738476 00000100:00000001:0.0:1788572599.459319:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.459326:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.459332:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.459336:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.459341:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a3c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.459348:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.459352:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.459494:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.459498:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.459500:0:266400:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572599.459502:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073b20 00000080:00200000:0.0:1788572599.459505:0:266400:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572599.459511:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00800000:00000002:0.0:1788572599.459517:0:266400:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572599.459523:0:266400:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.459527:0:266400:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.459530:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073a40 00800000:00000002:0.0:1788572599.459535:0:266400:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.459539:0:266400:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.459542:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.459549:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.459551:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.459553:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.459555:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.459557:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.459559:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.459562:0:266400:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572599.459568:0:266400:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.459571:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073910 00000002:00010000:0.0:1788572599.459573:0:266400:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572599.459578:0:266400:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572599.459581:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.459585:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.459590:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46800 count: 2 00010000:00000001:0.0:1788572599.459594:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.459600:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.459603:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572599.459607:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 02000000:00000001:0.0:1788572599.459616:0:266400:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572599.459619:0:266400:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.459622:0:266400:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00000001:0.0:1788572599.459628:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572599.459631:0:266400:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572599.459633:0:266400:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.459696:0:266400:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.459699:0:266400:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.459702:0:266400:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572599.459705:0:266400:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572599.459709:0:266400:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572599.459711:0:266400:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.459722:0:266400:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572599.459724:0:266400:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.459727:0:266400:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572599.459731:0:266400:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093168923 00800000:00000001:0.0:1788572599.459736:0:266400:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.459738:0:266400:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.459741:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.459743:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.459746:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=3 00000020:00000001:0.0:1788572599.459749:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.459754:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.459760:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.459762:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.459764:0:266400:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.459766:0:266400:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.459768:0:266400:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168923 from it ffffb95bc2073b58 00010000:00000001:0.0:1788572599.459770:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.459771:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.459772:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=3 00000020:00000001:0.0:1788572599.459774:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.459776:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.459779:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.459781:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572599.459788:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.459789:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.459790:0:266400:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.459792:0:266400:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.459794:0:266400:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00000001:0.0:1788572599.459800:0:266400:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.459808:0:266400:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.459810:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.459813:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.459814:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.459816:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459818:0:266400:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.459820:0:266400:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459822:0:266400:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459825:0:266400:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459827:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.459828:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.459829:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572599.459831:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572599.459832:0:266400:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.459835:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572599.459838:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.459841:0:266400:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572599.459842:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073ae8 00000080:00002000:0.0:1788572599.459843:0:266400:0:(dcache.c:87:ll_dcompare()) found name d801b.sanity(ffff9c37034e2510) flags 0x20000e refc 6 00000080:00000001:0.0:1788572599.459846:0:266400:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.459847:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073b30 00000080:00200000:0.0:1788572599.459848:0:266400:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d801b.sanity, flags=24 00000080:00000001:0.0:1788572599.459851:0:266400:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.459852:0:266400:0:(llite_lib.c:3161:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.459855:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.459856:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.459858:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572599.459866:0:266400:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f2913a0. 00000020:00000001:0.0:1788572599.459869:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073a88 00000080:00200000:0.0:1788572599.459870:0:266400:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d4, dir=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), flags=24 00000080:00000001:0.0:1788572599.459873:0:266400:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572599.459875:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073938 00000080:00200000:0.0:1788572599.459889:0:266400:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d4, dir=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), intent=0 00000080:00000010:0.0:1788572599.459896:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.459900:0:266400:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.459902:0:266400:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.459904:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073858 00800000:00000002:0.0:1788572599.459911:0:266400:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'lookup' for [0x0:0x0:0x0] 'd4' on [0x200002b17:0xdf9:0x0] 00800000:00000001:0.0:1788572599.459914:0:266400:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.459916:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.459917:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.459918:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.459920:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.459922:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.459924:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdf9:0x0] 00800000:00000001:0.0:1788572599.459926:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.459927:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20737f0 00800000:00000002:0.0:1788572599.459929:0:266400:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xdf9:0x0], fid2=[0x0:0x0:0x0], name='d4' -> mds #0 00000002:00000001:0.0:1788572599.459932:0:266400:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.459933:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073728 00000002:00010000:0.0:1788572599.459935:0:266400:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d4,[0x0:0x0:0x0]) in obj [0x200002b17:0xdf9:0x0], intent: lookup flags 0 00000002:00000001:0.0:1788572599.459938:0:266400:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.459940:0:266400:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572599.459944:0:266400:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572599.459947:0:266400:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572599.459950:0:266400:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.459952:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.459955:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.459956:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.459957:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.459960:0:266400:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.459962:0:266400:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.459964:0:266400:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.459969:0:266400:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572599.459973:0:266400:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459976:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.459978:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459980:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.459981:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.459984:0:266400:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.459988:0:266400:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00010000:00000001:0.0:1788572599.459990:0:266400:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.459992:0:266400:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.460000:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46d00 count: 5 00010000:00000001:0.0:1788572599.460002:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.460004:0:266400:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.460007:0:266400:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31800. 00000020:00000001:0.0:1788572599.460010:0:266400:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.460013:0:266400:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31800 with handle 0x1441586093168946 to hash 00000020:00000001:0.0:1788572599.460015:0:266400:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.460016:0:266400:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.460019:0:266400:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.460021:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.460023:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.460026:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.460033:0:266400:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x0 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.460042:0:266400:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x0 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.460047:0:266400:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.460048:0:266400:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.460053:0:266400:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a780. 00000100:00000001:0.0:1788572599.460055:0:266400:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040026496 : -109714669525120 : ffff9c371018a780) 00000100:00000001:0.0:1788572599.460056:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.460058:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000001:0.0:1788572599.460060:0:266400:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.460062:0:266400:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.460063:0:266400:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.460064:0:266400:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.460068:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.460074:0:266400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.460076:0:266400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.460085:0:266400:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.460087:0:266400:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.460089:0:266400:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.460093:0:266400:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job rm:5fa795d2-07a0-4260-8bbc-bb3cde572406:266400:1875440809984768:192.168.202.148@tcp:101:rm.0 00000100:00000001:0.0:1788572599.460096:0:266400:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.460098:0:266400:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.460100:0:266400:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.460102:0:266400:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.460107:0:266400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572599.460109:0:266400:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.460113:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.460117:0:266400:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8840. 00000400:00000010:0.0:1788572599.460121:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572599.460124:0:266400:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809984768, portal 10 00000100:00000001:0.0:1788572599.460125:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.460126:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572599.460131:0:266400:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.460138:0:266400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.460142:0:266400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.460145:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572599.460148:0:266400:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1875440809984768, offset 0 00000400:00000200:0.0:1788572599.460154:0:266400:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.460160:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.460165:0:266400:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.460168:0:266400:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.460171:0:266400:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.460174:0:266400:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.460177:0:266400:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.460182:0:266400:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540050:540050:256:4294967295] 192.168.202.148@tcp LPNI seq info [540015:540015:8:4294967295] 00000400:00000200:0.0:1788572599.460186:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.460196:0:266400:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.460208:0:266400:0:(socklnd_cb.c:931:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.460213:0:266400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788572599.460218:0:266400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.460224:0:266400:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.460227:0:266400:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0b00 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.460243:0:266400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.460246:0:266400:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15072 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.460250:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.460252:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738482 00000100:00000001:0.0:1788572599.460257:0:266400:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307738481 00000100:00000001:0.0:1788572599.460261:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738482 00000100:00000001:0.0:1788572599.460264:0:266400:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307738481 00000100:00000001:0.0:1788572599.460267:0:266400:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.460269:0:266400:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.460271:0:266400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.460273:0:266400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.460275:0:266400:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a780 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.460277:0:266400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.460279:0:266400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.470772:0:266400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.470781:0:266400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.470784:0:266400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.470788:0:266400:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8840. 00000400:00000200:0.0:1788572599.470793:0:266400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572599.470795:0:266400:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572599.470798:0:266400:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.470806:0:266400:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.470821:0:266400:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.470834:0:266400:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.470845:0:266400:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.470851:0:266400:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.470855:0:266400:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.470860:0:266400:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.470864:0:266400:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.470870:0:266400:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.470894:0:266400:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.470901:0:266400:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=45 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572599.470907:0:266400:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572599.470910:0:266400:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.470914:0:266400:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572599.470918:0:266400:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.470923:0:266400:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.470926:0:266400:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572599.470931:0:266400:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.470934:0:266400:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.470945:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.470963:0:266400:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.470970:0:266400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.470974:0:266400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.470984:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.471004:0:266400:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job rm:5fa795d2-07a0-4260-8bbc-bb3cde572406:266400:1875440809984768:192.168.202.148@tcp:101:rm.0 00000100:00000001:0.0:1788572599.471011:0:266400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.471016:0:266400:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.471019:0:266400:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.471022:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.471030:0:266400:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.471042:0:266400:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.471048:0:266400:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a780. 00000100:00000001:0.0:1788572599.471055:0:266400:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.471065:0:266400:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.471068:0:266400:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.471073:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471077:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471081:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=4 00000020:00000001:0.0:1788572599.471084:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471088:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000040:0.0:1788572599.471098:0:266400:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372db31800, remote cookie: 0xd9894672d99a2fbf, flags: 0x1001 00010000:00010000:0.0:1788572599.471106:0:266400:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572599.471121:0:266400:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdfc:0x0].0x0, instead of [0x200002b17:0xdf9:0x0].0x0 00010000:00000001:0.0:1788572599.471128:0:266400:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572599.471139:0:266400:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46f00. 00010000:00000010:0.0:1788572599.471146:0:266400:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676d80. 00010000:00000001:0.0:1788572599.471155:0:266400:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.471167:0:266400:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.471192:0:266400:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471212:0:266400:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572599.471218:0:266400:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572599.471223:0:266400:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572599.471229:0:266400:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572599.471233:0:266400:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572599.471238:0:266400:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdfc:0x0].0x0 (ffff9c3713c46f00) refcount = 1 00010000:00010000:0.0:1788572599.471247:0:266400:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471292:0:266400:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572599.471298:0:266400:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572599.471301:0:266400:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.471308:0:266400:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572599.471313:0:266400:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.471316:0:266400:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471324:0:266400:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572599.471326:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471328:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.471331:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471332:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.471334:0:266400:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.471340:0:266400:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572599.471343:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471347:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471350:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.471352:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471355:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471358:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471359:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471360:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=4 00000020:00000001:0.0:1788572599.471362:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471365:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471367:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.471370:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.471373:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471380:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471386:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.471388:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471389:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471390:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=5 00000020:00000001:0.0:1788572599.471392:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471394:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471396:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.471399:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471405:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471407:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.471409:0:266400:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471416:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.471417:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471418:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.471419:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471421:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572599.471427:0:266400:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572599.471444:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471446:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471448:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.471451:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471457:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471462:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471467:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.471471:0:266400:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.471477:0:266400:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.471484:0:266400:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572599.471488:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471493:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471512:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.471518:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471527:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00000002:00010000:0.0:1788572599.471536:0:266400:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.471571:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471575:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.471579:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.471584:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.471588:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.471592:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.471598:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.471604:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000040:0.0:1788572599.471613:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46f00 count: 2 00010000:00000001:0.0:1788572599.471619:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572599.471661:0:266400:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3580 (0 0) 00010000:00000001:0.0:1788572599.471671:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.471674:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.471677:0:266400:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572599.471682:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20736b0 00000002:00002000:0.0:1788572599.471688:0:266400:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=d4 intent=lookup status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572599.471696:0:266400:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.471703:0:266400:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.471710:0:266400:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572599.471717:0:266400:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200002b17:0xdf9:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572599.471726:0:266400:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.471733:0:266400:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572599.471737:0:266400:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc2073a28 it_disposition b 00000080:00000001:0.0:1788572599.471743:0:266400:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572599.471752:0:266400:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572599.471757:0:266400:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572599.471762:0:266400:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329148 : 144115373145329148 : 200002b17000dfc) 00000080:00000001:0.0:1788572599.471768:0:266400:0:(namei.c:113:ll_iget()) Process entered 00000100:00000001:0.0:1788572599.471789:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.471800:0:266397:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.471805:0:266397:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.471809:0:266397:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788572599.471815:0:266397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31dd0 00000400:00000010:0.0:1788572599.471821:0:266397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31dd0. 00000100:00000001:0.0:1788572599.471828:0:266397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.471836:0:266397:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.471889:0:266397:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.471915:0:266397:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.471918:0:266397:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.471923:0:266397:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.471927:0:266397:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.471930:0:266397:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.471934:0:266397:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.471939:0:266397:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.471947:0:266397:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.471954:0:266397:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=45 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572599.471963:0:266397:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572599.471967:0:266397:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.471979:0:266397:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572599.471984:0:266397:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.471991:0:266397:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.471996:0:266397:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572599.472002:0:266397:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.472006:0:266397:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.472017:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.472045:0:266397:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.472052:0:266397:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.472056:0:266397:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.472068:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.472129:0:266397:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809984512:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.472142:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.472147:0:266397:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.472152:0:266397:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.472157:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.472167:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.472198:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.472206:0:266397:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.472213:0:266397:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.472217:0:266397:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.472224:0:266397:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.472230:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472234:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472239:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=4 00000020:00000001:0.0:1788572599.472246:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472253:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000040:0.0:1788572599.472261:0:266397:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372db30c00, remote cookie: 0xd9894672d99a2f9c, flags: 0x1001 00010000:00010000:0.0:1788572599.472270:0:266397:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xdf9:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572599.472297:0:266397:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdfa:0x0].0x0, instead of [0x200002b17:0xdf9:0x0].0x0 00010000:00000001:0.0:1788572599.472308:0:266397:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572599.472315:0:266397:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46500. 00010000:00000010:0.0:1788572599.472322:0:266397:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676700. 00010000:00000001:0.0:1788572599.472330:0:266397:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.472337:0:266397:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.472392:0:266397:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472412:0:266397:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572599.472417:0:266397:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572599.472421:0:266397:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572599.472424:0:266397:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572599.472428:0:266397:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572599.472433:0:266397:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdfa:0x0].0x0 (ffff9c3713c46500) refcount = 1 00010000:00010000:0.0:1788572599.472442:0:266397:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472465:0:266397:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572599.472469:0:266397:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572599.472473:0:266397:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.472479:0:266397:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572599.472485:0:266397:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.472491:0:266397:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472514:0:266397:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572599.472519:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472523:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.472526:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472530:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.472534:0:266397:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.472541:0:266397:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572599.472545:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472549:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472553:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.472559:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472565:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472578:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472581:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472585:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=4 00000020:00000001:0.0:1788572599.472591:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472598:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472605:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.472610:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.472617:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472657:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472660:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.472665:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472669:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472673:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=5 00000020:00000001:0.0:1788572599.472678:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472685:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472689:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.472695:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472710:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472714:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.472719:0:266397:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472740:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.472744:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472748:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.472752:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472755:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572599.472766:0:266397:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572599.472798:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472802:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472805:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.472810:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472815:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472820:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472823:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.472827:0:266397:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.472833:0:266397:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.472837:0:266397:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572599.472840:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472843:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472847:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.472853:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472858:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00000002:00010000:0.0:1788572599.472866:0:266397:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.472895:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472899:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.472902:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.472906:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.472910:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.472914:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.472918:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.472923:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000040:0.0:1788572599.472930:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46500 count: 2 00010000:00000001:0.0:1788572599.472936:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572599.472944:0:266397:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3578 (0 0) 00010000:00000001:0.0:1788572599.472952:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.472956:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.472965:0:266397:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572599.472969:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf750 00000002:00002000:0.0:1788572599.472975:0:266397:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=d1 intent=lookup status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572599.472981:0:266397:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.472987:0:266397:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.472993:0:266397:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572599.472999:0:266397:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200002b17:0xdf9:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572599.473008:0:266397:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.473014:0:266397:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572599.473018:0:266397:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc1eafac8 it_disposition b 00000080:00000001:0.0:1788572599.473022:0:266397:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572599.473026:0:266397:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572599.473033:0:266397:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572599.473037:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329146 : 144115373145329146 : 200002b17000dfa) 00000080:00000001:0.0:1788572599.473045:0:266397:0:(namei.c:113:ll_iget()) Process entered 00000080:00000001:0.0:1788572599.473052:0:266397:0:(llite_lib.c:1957:ll_update_lsm_md()) Process entered 00000080:00000002:0.0:1788572599.473056:0:266397:0:(llite_lib.c:1961:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200002b17:0xdfa:0x0] 00000080:00000001:0.0:1788572599.473063:0:266397:0:(llite_lib.c:1978:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.473069:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329146 : 144115373145329146 : 200002b17000dfa) 00000080:00000001:0.0:1788572599.473075:0:266397:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572599.473081:0:266397:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200002b17:0xdfa:0x0], updating i_size 4096 00000080:00200000:0.0:1788572599.473087:0:266397:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdfa:0x0](ffff9c37081ff448): rc = 0 00000080:00000001:0.0:1788572599.473095:0:266397:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358906287176 : -109714803264440 : ffff9c37081ff448) 00010000:00000001:0.0:1788572599.473100:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.473104:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.473108:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.473115:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473122:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473126:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473130:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.473134:0:266397:0:(llite_lib.c:1912:ll_update_default_lsm_md()) Process entered 00000080:00000001:0.0:1788572599.473138:0:266397:0:(llite_lib.c:1925:ll_update_default_lsm_md()) Process leaving 00000080:00000001:0.0:1788572599.473142:0:266397:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572599.473148:0:266397:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572599.473153:0:266397:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.473165:0:266397:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdfa:0x0](ffff9c37081ff448) for lock 0x1441586093168938 00800000:00000001:0.0:1788572599.473172:0:266397:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.473176:0:266397:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.473179:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.473183:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.473188:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.473194:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473201:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473208:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473211:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.473215:0:266397:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.473220:0:266397:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572599.473228:0:266397:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37034e21b0 inode ffff9c37081ff448 refc 1 flags 0x20000e 00000080:00000001:0.0:1788572599.473236:0:266397:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.473239:0:266397:0:(llite_lib.c:3165:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.473245:0:266397:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572599.473252:0:266397:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdfa:0x0](ffff9c37081ff448) 00000080:00010000:0.0:1788572599.473260:0:266397:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168938 from it ffffb95bc1eafac8 00010000:00000001:0.0:1788572599.473265:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.473268:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.473271:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=3 00000020:00000001:0.0:1788572599.473276:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473280:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788572599.473285:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.473290:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.473312:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473317:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.473322:0:266397:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.473326:0:266397:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.473331:0:266397:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30c00/0x1441586093168938 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdfa:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f9c expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.473372:0:266397:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.473376:0:266397:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.473380:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.473384:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473388:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.473393:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473398:0:266397:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.473402:0:266397:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473409:0:266397:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473416:0:266397:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473422:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.473427:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473431:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.473435:0:266397:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572599.473443:0:266397:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20600. 00000100:00000001:0.0:1788572599.473455:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.473464:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809984512/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.473494:0:266397:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.473497:0:266397:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.473501:0:266397:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572599.473509:0:266397:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.473512:0:266397:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.473516:0:266397:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.473523:0:266397:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.473526:0:266397:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788572599.473532:0:266397:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.473536:0:266397:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.473541:0:266397:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572599.473548:0:266397:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.473552:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.473558:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.473563:0:266397:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdfa:0x0](ffff9c37081ff448), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.473573:0:266397:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572599.473593:0:266397:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f291760. 00000080:00000001:0.0:1788572599.473599:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.473604:0:266397:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdfa:0x0](ffff9c37081ff448), inode mode 41ed mask 3 00000080:00000001:0.0:1788572599.473614:0:266397:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.473620:0:266397:0:(namei.c:1405:ll_atomic_open()) Process entered 00000020:00000001:0.0:1788572599.473624:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafb58 00000080:00200000:0.0:1788572599.473631:0:266397:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f13, dir=[0x200002b17:0xdfa:0x0](ffff9c37081ff448), file ffff9c3708c2a480, open_flags 8941, mode 81b6 opened 0 00000080:00000010:0.0:1788572599.473668:0:266397:0:(namei.c:1432:ll_atomic_open()) kmalloced '(it)': 72 at ffff9c3711626180. 00000080:00000001:0.0:1788572599.473675:0:266397:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572599.473680:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafa08 00000080:00200000:0.0:1788572599.473684:0:266397:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f13, dir=[0x200002b17:0xdfa:0x0](ffff9c37081ff448), intent=open|creat 00000080:00000010:0.0:1788572599.473694:0:266397:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20600. 00000080:00000001:0.0:1788572599.473703:0:266397:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.473708:0:266397:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.473712:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf928 00800000:00000002:0.0:1788572599.473717:0:266397:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'open|creat' for [0x0:0x0:0x0] 'f13' on [0x200002b17:0xdfa:0x0] 00800000:00000001:0.0:1788572599.473726:0:266397:0:(lmv_intent.c:299:lmv_intent_open()) Process entered 00800000:00000001:0.0:1788572599.473732:0:266397:0:(lmv_obd.c:2248:lmv_locate_tgt_create()) Process entered 00800000:00000001:0.0:1788572599.473738:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.473741:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.473745:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.473750:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.473755:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.473762:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfa:0x0] 00800000:00000001:0.0:1788572599.473769:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.473775:0:266397:0:(lmv_obd.c:2294:lmv_locate_tgt_create()) Process leaving (rc=18446634359068665856 : -109714640885760 : ffff9c3711cda800) 00800000:00000001:0.0:1788572599.473781:0:266397:0:(lmv_obd.c:1208:lmv_fid_alloc()) Process entered 00800000:00000001:0.0:1788572599.473786:0:266397:0:(obd_class.h:931:obd_fid_alloc()) Process entered 00000002:00000001:0.0:1788572599.473790:0:266397:0:(mdc_request.c:2962:mdc_fid_alloc()) Process entered 40000000:00000001:0.0:1788572599.473795:0:266397:0:(fid_request.c:299:seq_client_alloc_fid()) Process entered 40000000:00000040:0.0:1788572599.473800:0:266397:0:(fid_request.c:336:seq_client_alloc_fid()) cli-cli-lustre-MDT0000-mdc-ffff9c3711155800: Allocated FID [0x200002b17:0xdff:0x0] 40000000:00000001:0.0:1788572599.473807:0:266397:0:(fid_request.c:340:seq_client_alloc_fid()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.473810:0:266397:0:(mdc_request.c:2968:mdc_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.473816:0:266397:0:(obd_class.h:944:obd_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.473819:0:266397:0:(lmv_obd.c:1232:lmv_fid_alloc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.473825:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf8a8 00800000:00000002:0.0:1788572599.473837:0:266397:0:(lmv_intent.c:382:lmv_intent_open()) OPEN_INTENT with fid1=[0x200002b17:0xdfa:0x0], fid2=[0x200002b17:0xdff:0x0], name='f13' -> mds #0 layout version 1 00000002:00000001:0.0:1788572599.473846:0:266397:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.473851:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf7e0 00000002:00010000:0.0:1788572599.473856:0:266397:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f13,[0x200002b17:0xdff:0x0]) in obj [0x200002b17:0xdfa:0x0], intent: open|creat flags 0104502 00000002:00000001:0.0:1788572599.473866:0:266397:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.473869:0:266397:0:(mdc_locks.c:258:mdc_intent_open_pack()) Process entered 00000002:00000001:0.0:1788572599.473873:0:266397:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00000002:00000001:0.0:1788572599.473890:0:266397:0:(mdc_reint.c:67:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.473895:0:266397:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572599.473901:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46500 count: 2 00010000:00000001:0.0:1788572599.473907:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.473914:0:266397:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572599.473919:0:266397:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473923:0:266397:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.473928:0:266397:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.473936:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572599.473942:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572599.473950:0:266397:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572599.473955:0:266397:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.473961:0:266397:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.473965:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.473972:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.473976:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.473980:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.473986:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.473991:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.473996:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.474005:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572599.474012:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.474018:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.474024:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.474029:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.474033:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.474039:0:266397:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.474051:0:266397:0:(mdc_locks.c:430:mdc_intent_open_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572599.474064:0:266397:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.474067:0:266397:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.474072:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46500 count: 2 00010000:00000001:0.0:1788572599.474076:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.474082:0:266397:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.474087:0:266397:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788572599.474093:0:266397:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.474098:0:266397:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168969 to hash 00000020:00000001:0.0:1788572599.474105:0:266397:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.474108:0:266397:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572599.474115:0:266397:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572599.474120:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.474124:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.474132:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.474149:0:266397:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788572599.474168:0:266397:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788572599.474179:0:266397:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788572599.474203:0:266397:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.474217:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.474221:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.474227:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.474232:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040025536 : -109714669526080 : ffff9c371018a3c0) 00000100:00000001:0.0:1788572599.474237:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.474242:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572599.474249:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.474260:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.474266:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.474269:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.474277:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 0 ref 2 fl New:PQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.474307:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.474312:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.474319:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.474324:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.474331:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.474340:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809985664:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.474347:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.474352:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.474356:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.474361:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.474409:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572599.474414:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.474426:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.474431:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788572599.474438:0:266397:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff9c372d20e800. 00000100:00000200:0.0:1788572599.474445:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1875440809985664, portal 10 00000100:00000001:0.0:1788572599.474450:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.474453:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572599.474464:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572684 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.474485:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.474490:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.474494:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572599.474499:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 912 bytes to portal 12, xid 1875440809985664, offset 0 00000400:00000200:0.0:1788572599.474512:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.474520:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.474529:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.474535:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.474550:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.474558:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.474566:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.474574:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540055:540055:255:4294967295] 192.168.202.148@tcp LPNI seq info [540020:540020:7:4294967295] 00000400:00000200:0.0:1788572599.474584:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.474602:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.474614:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 912 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.474623:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788572599.474662:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.474674:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.474681:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0700 type 1, nob 1008 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.474690:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.474696:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15067 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.474708:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.474714:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738497 00000100:00000001:0.0:1788572599.474722:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307738506 00000100:00000001:0.0:1788572599.474728:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738497 00000100:00000001:0.0:1788572599.474734:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307738506 00000100:00000001:0.0:1788572599.474741:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.474747:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.474754:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.474759:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.474765:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a3c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.474772:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.474776:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.474792:0:266400:0:(llite_lib.c:1957:ll_update_lsm_md()) Process entered 00000080:00000002:0.0:1788572599.474797:0:266400:0:(llite_lib.c:1961:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200002b17:0xdfc:0x0] 00000080:00000001:0.0:1788572599.474803:0:266400:0:(llite_lib.c:1978:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.474812:0:266400:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329148 : 144115373145329148 : 200002b17000dfc) 00000080:00000001:0.0:1788572599.474819:0:266400:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572599.474833:0:266400:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200002b17:0xdfc:0x0], updating i_size 4096 00000080:00200000:0.0:1788572599.474841:0:266400:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdfc:0x0](ffff9c37081f80c8): rc = 0 00000080:00000001:0.0:1788572599.474846:0:266400:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358906257608 : -109714803294008 : ffff9c37081f80c8) 00010000:00000001:0.0:1788572599.474853:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.474856:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.474861:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.474866:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.474872:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.474890:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.474894:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.474897:0:266400:0:(llite_lib.c:1912:ll_update_default_lsm_md()) Process entered 00000080:00000001:0.0:1788572599.474901:0:266400:0:(llite_lib.c:1925:ll_update_default_lsm_md()) Process leaving 00000080:00000001:0.0:1788572599.474905:0:266400:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572599.474914:0:266400:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572599.474918:0:266400:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.474924:0:266400:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdfc:0x0](ffff9c37081f80c8) for lock 0x1441586093168946 00800000:00000001:0.0:1788572599.474931:0:266400:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.474935:0:266400:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.474938:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.474941:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.474946:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.474952:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.474958:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.474963:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.474965:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.474968:0:266400:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.474973:0:266400:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572599.474980:0:266400:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37034e2000 inode ffff9c37081f80c8 refc 1 flags 0x20000e 00000080:00000001:0.0:1788572599.474985:0:266400:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.474988:0:266400:0:(llite_lib.c:3165:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.474992:0:266400:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572599.474998:0:266400:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdfc:0x0](ffff9c37081f80c8) 00000080:00010000:0.0:1788572599.475006:0:266400:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168946 from it ffffb95bc2073a28 00010000:00000001:0.0:1788572599.475010:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.475019:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.475022:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=3 00000020:00000001:0.0:1788572599.475027:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.475034:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788572599.475041:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.475047:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.475065:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.475069:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.475070:0:266400:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.475074:0:266400:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.475079:0:266400:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31800/0x1441586093168946 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2fbf expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.475102:0:266400:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.475107:0:266400:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.475111:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.475116:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.475120:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.475123:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475127:0:266400:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.475132:0:266400:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475137:0:266400:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475144:0:266400:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475150:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.475154:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.475158:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.475162:0:266400:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572599.475168:0:266400:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20000. 00000100:00000001:0.0:1788572599.475178:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.475188:0:266400:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5c00 x1875440809984768/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/584 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.475212:0:266400:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.475216:0:266400:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.475226:0:266400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572599.475231:0:266400:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.475234:0:266400:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.475236:0:266400:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.475242:0:266400:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.475246:0:266400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572599.475253:0:266400:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.475257:0:266400:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.475261:0:266400:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5c00. 00000100:00000001:0.0:1788572599.475266:0:266400:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.475270:0:266400:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.475277:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.475282:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.475291:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572599.475301:0:266400:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff9c371f291cc0. 00000020:00000001:0.0:1788572599.475307:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073ae8 00000080:00200000:0.0:1788572599.475312:0:266400:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f12, dir=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8), flags=8 00000080:00000001:0.0:1788572599.475318:0:266400:0:(namei.c:1066:ll_lookup_it()) Process entered 00000020:00000001:0.0:1788572599.475322:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073998 00000080:00200000:0.0:1788572599.475326:0:266400:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f12, dir=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8), intent=getattr 00000080:00000010:0.0:1788572599.475336:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.475341:0:266400:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.475345:0:266400:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.475349:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20738b8 00800000:00000002:0.0:1788572599.475354:0:266400:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x0:0x0:0x0] 'f12' on [0x200002b17:0xdfc:0x0] 00800000:00000001:0.0:1788572599.475362:0:266400:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.475366:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.475369:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.475371:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.475375:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.475380:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.475386:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfc:0x0] 00800000:00000001:0.0:1788572599.475390:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.475394:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073850 00800000:00000002:0.0:1788572599.475399:0:266400:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xdfc:0x0], fid2=[0x0:0x0:0x0], name='f12' -> mds #0 00000002:00000001:0.0:1788572599.475408:0:266400:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.475417:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073788 00000002:00010000:0.0:1788572599.475422:0:266400:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f12,[0x0:0x0:0x0]) in obj [0x200002b17:0xdfc:0x0], intent: getattr flags 01 00000002:00000001:0.0:1788572599.475430:0:266400:0:(mdc_locks.c:1027:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1788572599.475433:0:266400:0:(mdc_locks.c:623:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1788572599.475438:0:266400:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572599.475445:0:266400:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=8 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572599.475452:0:266400:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.475457:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.475463:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.475466:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.475470:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.475476:0:266400:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.475479:0:266400:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.475485:0:266400:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.475493:0:266400:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572599.475501:0:266400:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475506:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.475510:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475515:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.475518:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.475521:0:266400:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.475527:0:266400:0:(mdc_locks.c:694:mdc_intent_getattr_pack()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00010000:00000001:0.0:1788572599.475533:0:266400:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.475536:0:266400:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.475542:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46f00 count: 2 00010000:00000001:0.0:1788572599.475547:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.475554:0:266400:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.475560:0:266400:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788572599.475565:0:266400:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.475569:0:266400:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x144158609316897e to hash 00000020:00000001:0.0:1788572599.475574:0:266400:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.475579:0:266400:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.475583:0:266400:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.475587:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.475589:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.475600:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.475612:0:266400:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.475633:0:266400:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: --/CR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266400 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.475673:0:266400:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.475678:0:266400:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.475683:0:266400:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a780. 00000100:00000001:0.0:1788572599.475686:0:266400:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040026496 : -109714669525120 : ffff9c371018a780) 00000100:00000001:0.0:1788572599.475688:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.475689:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000001:0.0:1788572599.475692:0:266400:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.475695:0:266400:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.475696:0:266400:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.475698:0:266400:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.475703:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.475709:0:266400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.475710:0:266400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.475712:0:266400:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.475713:0:266400:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.475715:0:266400:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.475717:0:266400:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job rm:5fa795d2-07a0-4260-8bbc-bb3cde572406:266400:1875440809985920:192.168.202.148@tcp:101:rm.0 00000100:00000001:0.0:1788572599.475720:0:266400:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.475722:0:266400:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.475724:0:266400:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.475730:0:266400:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.475734:0:266400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572599.475739:0:266400:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.475748:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.475758:0:266400:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8840. 00000400:00000010:0.0:1788572599.475765:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572599.475771:0:266400:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809985920, portal 10 00000100:00000001:0.0:1788572599.475776:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.475780:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572599.475790:0:266400:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.475805:0:266400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.475813:0:266400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.475820:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572599.475826:0:266400:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1875440809985920, offset 0 00000400:00000200:0.0:1788572599.475835:0:266400:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.475842:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.475848:0:266400:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.475854:0:266400:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.475859:0:266400:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.475864:0:266400:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.475869:0:266400:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.475874:0:266400:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540058:540058:256:4294967295] 192.168.202.148@tcp LPNI seq info [540023:540023:8:4294967295] 00000400:00000200:0.0:1788572599.475894:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.475902:0:266400:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.475907:0:266400:0:(socklnd_cb.c:931:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.475913:0:266400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572599.475921:0:266400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.475930:0:266400:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.475935:0:266400:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.475954:0:266400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.475960:0:266400:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15067 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.475966:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.475970:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738498 00000100:00000001:0.0:1788572599.475974:0:266400:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307738508 00000100:00000001:0.0:1788572599.475985:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738498 00000100:00000001:0.0:1788572599.475990:0:266400:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307738508 00000100:00000001:0.0:1788572599.475995:0:266400:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.476000:0:266400:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.476005:0:266400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.476008:0:266400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.476012:0:266400:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a780 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.476016:0:266400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.476019:0:266400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.480760:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.480769:0:266397:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.480774:0:266397:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.480781:0:266397:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788572599.480791:0:266397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372d20e800 00000400:00000010:0.0:1788572599.480798:0:266397:0:(lib-lnet.h:286:lnet_md_free()) kfreed 'md': 632 at ffff9c372d20e800. 00000100:00000001:0.0:1788572599.480807:0:266397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.480819:0:266397:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.480852:0:266397:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RPQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.480870:0:266397:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.480886:0:266397:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.480889:0:266397:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.480891:0:266397:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.480895:0:266397:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.480903:0:266397:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.480912:0:266397:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.480925:0:266397:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.480939:0:266397:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=45 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572599.480949:0:266397:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1788572599.480966:0:266397:0:(client.c:1517:ptlrpc_check_status()) @@@ check status: rc = 301 req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/608 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RPQU/600/0 rc 0/301 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.480977:0:266397:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=301 : 301 : 12d) 00000100:00000001:0.0:1788572599.481001:0:266397:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.481009:0:266397:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572599.481013:0:266397:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.481020:0:266397:0:(client.c:1805:after_reply()) Process leaving (rc=301 : 301 : 12d) 00000100:00000040:0.0:1788572599.481035:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/608 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RPQU/600/0 rc 301/301 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.481065:0:266397:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=301 : 301 : 0x12d) 00000100:00000001:0.0:1788572599.481071:0:266397:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.481076:0:266397:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.481094:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/608 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RPQU/600/0 rc 301/301 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.481148:0:266397:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809985664:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572599.481158:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.481170:0:266397:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=301 : 301 : 12d) 00000100:00000001:0.0:1788572599.481175:0:266397:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.481182:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.481192:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/608 e 0 to 0 dl 1788572684 ref 2 fl Complete:RPQU/600/0 rc 301/301 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.481207:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.481212:0:266397:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.481217:0:266397:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.481219:0:266397:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=301 : 301 : 12d) 00010000:00000001:0.0:1788572599.481223:0:266397:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.481226:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.481231:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.481236:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788572599.481240:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572599.481243:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788572599.481247:0:266397:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481256:0:266397:0:(ldlm_request.c:719:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=301 : 301 : 0x12d) 00010000:00010000:0.0:1788572599.481270:0:266397:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481281:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.481286:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481295:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.481297:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.481300:0:266397:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 3/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481306:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.481310:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.481315:0:266397:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572599.481318:0:266397:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.481327:0:266397:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1a06630) 00000080:00000001:0.0:1788572599.481334:0:266397:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00010000:00010000:0.0:1788572599.481341:0:266397:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481366:0:266397:0:(ldlm_inodebits.c:510:ldlm_cli_inodebits_convert()) Process entered 00010000:00000001:0.0:1788572599.481375:0:266397:0:(ldlm_inodebits.c:533:ldlm_cli_inodebits_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00010000:0.0:1788572599.481384:0:266397:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 4/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481399:0:266397:0:(ldlm_request.c:1743:ldlm_cli_convert()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00010000:00000001:0.0:1788572599.481404:0:266397:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572599.481406:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.481407:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.481412:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788572599.481415:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572599.481430:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788572599.481440:0:266397:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572599.481447:0:266397:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 5/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000080:00000001:0.0:1788572599.481461:0:266397:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572599.481466:0:266397:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.481476:0:266397:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788572599.481480:0:266397:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572599.481489:0:266397:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572599.481494:0:266397:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572599.481501:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.481503:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572599.481512:0:266397:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168969 from hash 00010000:00000001:0.0:1788572599.481516:0:266397:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572599.481519:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.481525:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.481529:0:266397:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572599.481532:0:266397:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572599.481535:0:266397:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788572599.481545:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.481550:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.481553:0:266397:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.481555:0:266397:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.481558:0:266397:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 3/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.481590:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.481592:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.481594:0:266397:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788572599.481601:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.481605:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.481607:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.481610:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572599.481613:0:266397:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31a00/0x1441586093168969 lrc: 0/0,0 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.483960:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.483965:0:266397:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=301 : 301 : 12d) 00000100:00000001:0.0:1788572599.483974:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.483987:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809985664/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/608 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.484009:0:266397:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.484011:0:266397:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.484014:0:266397:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572599.484038:0:266397:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.484041:0:266397:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.484048:0:266397:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=6 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.484059:0:266397:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.484063:0:266397:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788572599.484071:0:266397:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.484075:0:266397:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.484080:0:266397:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572599.484088:0:266397:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.484097:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000002:00080000:0.0:1788572599.484109:0:266397:0:(mdc_locks.c:1157:mdc_enqueue_base()) lustre-MDT0000-mdc-ffff9c3711155800: resend:1 op:3 [0x200002b17:0xdfa:0x0]/[0x200002b17:0xdff:0x0] 00000002:00000001:0.0:1788572599.484115:0:266397:0:(mdc_locks.c:258:mdc_intent_open_pack()) Process entered 00000002:00000001:0.0:1788572599.484119:0:266397:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00000002:00000001:0.0:1788572599.484128:0:266397:0:(mdc_reint.c:67:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.484137:0:266397:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572599.484141:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46500 count: 2 00010000:00000001:0.0:1788572599.484149:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.484163:0:266397:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572599.484171:0:266397:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484176:0:266397:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.484184:0:266397:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.484194:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572599.484203:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572599.484215:0:266397:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572599.484223:0:266397:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484240:0:266397:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.484243:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.484253:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.484257:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.484261:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484270:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.484275:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.484278:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.484286:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572599.484292:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484298:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.484306:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484314:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.484320:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484327:0:266397:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.484333:0:266397:0:(mdc_locks.c:430:mdc_intent_open_pack()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00010000:00000001:0.0:1788572599.484338:0:266397:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.484345:0:266397:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000040:0.0:1788572599.484347:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46500 count: 2 00010000:00000001:0.0:1788572599.484353:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.484373:0:266397:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.484378:0:266397:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788572599.484383:0:266397:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.484390:0:266397:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x144158609316898c to hash 00000020:00000001:0.0:1788572599.484399:0:266397:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.484402:0:266397:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788572599.484409:0:266397:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788572599.484412:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.484414:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.484418:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x144158609316898c lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.484438:0:266397:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x144158609316898c lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788572599.484472:0:266397:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788572599.484488:0:266397:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b6680 x1875440809986560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788572599.484505:0:266397:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db31400/0x144158609316898c lrc: 3/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 266397 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572599.484530:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.484535:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.484540:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a3c0. 00000100:00000001:0.0:1788572599.484547:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040025536 : -109714669526080 : ffff9c371018a3c0) 00000100:00000001:0.0:1788572599.484556:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.484559:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788572599.484570:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.484579:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484586:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.484593:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572599.484604:0:266397:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484612:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.484619:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572599.484629:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a3c0 going to sleep for 1 seconds 00000100:00000001:0.0:1788572599.484633:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.484660:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572599.484666:0:266397:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484676:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484694:0:266400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.484698:0:266400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.484703:0:266400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.484710:0:266400:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8840. 00000400:00000200:0.0:1788572599.484717:0:266400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572599.484726:0:266400:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572599.484733:0:266400:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.484745:0:266400:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.484832:0:266400:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.484851:0:266400:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.484859:0:266400:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.484872:0:266400:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.484891:0:266400:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.484898:0:266400:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.484901:0:266400:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.484906:0:266400:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.484912:0:266400:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.484929:0:266400:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=45 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572599.484942:0:266400:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572599.484947:0:266400:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.484949:0:266400:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572599.484954:0:266400:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.484965:0:266400:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.484971:0:266400:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572599.484976:0:266400:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.484980:0:266400:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.484990:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/640 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.485027:0:266400:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.485034:0:266400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.485039:0:266400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.485048:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/640 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.485070:0:266400:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job rm:5fa795d2-07a0-4260-8bbc-bb3cde572406:266400:1875440809985920:192.168.202.148@tcp:101:rm.0 00000100:00000001:0.0:1788572599.485083:0:266400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.485093:0:266400:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.485104:0:266400:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.485108:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.485125:0:266400:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/640 e 0 to 0 dl 1788572684 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.485164:0:266400:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.485174:0:266400:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c371018a780. 00000100:00000001:0.0:1788572599.485180:0:266400:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.485183:0:266400:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.485189:0:266400:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.485192:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485198:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485205:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788572599.485218:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485225:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000040:0.0:1788572599.485236:0:266400:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372db30400, remote cookie: 0xd9894672d99a308a, flags: 0x1001 00010000:00010000:0.0:1788572599.485247:0:266400:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: --/CR res: [0x200002b17:0xdfc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572599.485268:0:266400:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdfe:0x0].0x0, instead of [0x200002b17:0xdfc:0x0].0x0 00010000:00000001:0.0:1788572599.485286:0:266400:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572599.485296:0:266400:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46b00. 00010000:00000010:0.0:1788572599.485306:0:266400:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676300. 00010000:00000001:0.0:1788572599.485319:0:266400:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.485330:0:266400:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.485377:0:266400:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: --/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485398:0:266400:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572599.485402:0:266400:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572599.485404:0:266400:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572599.485408:0:266400:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572599.485421:0:266400:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572599.485425:0:266400:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdfe:0x0].0x0 (ffff9c3713c46b00) refcount = 1 00010000:00010000:0.0:1788572599.485439:0:266400:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485467:0:266400:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572599.485475:0:266400:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572599.485479:0:266400:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.485488:0:266400:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572599.485492:0:266400:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.485500:0:266400:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485522:0:266400:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572599.485526:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485528:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.485533:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485537:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.485542:0:266400:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.485552:0:266400:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572599.485559:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485564:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485573:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.485582:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485593:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485604:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485613:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485616:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788572599.485625:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485682:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485689:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.485693:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.485707:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485757:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485766:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.485773:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485779:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485786:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=5 00000020:00000001:0.0:1788572599.485791:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485797:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485802:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.485808:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 5/2,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485818:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485825:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.485828:0:266400:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485841:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.485844:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485846:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.485848:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485850:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572599.485858:0:266400:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/640 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1788572599.485874:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485889:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485892:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.485896:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485901:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00000002:00010000:0.0:1788572599.485906:0:266400:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572599.485929:0:266400:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372e9a9b40. 00010000:00000001:0.0:1788572599.485932:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485935:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.485937:0:266400:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.485940:0:266400:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.485944:0:266400:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1788572599.485947:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485949:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485951:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.485953:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485956:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00000002:00010000:0.0:1788572599.485962:0:266400:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.485972:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.485975:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.485977:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.485980:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.485983:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.485987:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.485990:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.485993:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000040:0.0:1788572599.485999:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 2 00010000:00000001:0.0:1788572599.486001:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572599.486007:0:266400:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3582 (0 0) 00010000:00000001:0.0:1788572599.486013:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486015:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.486017:0:266400:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572599.486021:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073710 00000002:00002000:0.0:1788572599.486025:0:266400:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f12 intent=getattr status=0 disp=0xb: rc = 0 00000002:00000001:0.0:1788572599.486030:0:266400:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.486039:0:266400:0:(lmv_intent.c:556:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.486043:0:266400:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572599.486048:0:266400:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xdfc:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572599.486054:0:266400:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.486059:0:266400:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572599.486061:0:266400:0:(namei.c:674:ll_lookup_it_finish()) it ffffb95bc2073b20 it_disposition b 00000080:00000001:0.0:1788572599.486065:0:266400:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572599.486069:0:266400:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572599.486072:0:266400:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572599.486076:0:266400:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329150 : 144115373145329150 : 200002b17000dfe) 00000080:00000001:0.0:1788572599.486080:0:266400:0:(namei.c:113:ll_iget()) Process entered 00000020:00000001:0.0:1788572599.486086:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.486090:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572599.486093:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572599.486096:0:266400:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572599.486100:0:266400:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572599.486104:0:266400:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572599.486111:0:266400:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626c60. 00020000:00000010:0.0:1788572599.486116:0:266400:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c372ecca780. 00020000:00000010:0.0:1788572599.486122:0:266400:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626d80. 00020000:00000001:0.0:1788572599.486125:0:266400:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061638528 : -109714647913088 : ffff9c3711626d80) 00020000:00000002:0.0:1788572599.486132:0:266400:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626d80, objid 0xdfe:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572599.486137:0:266400:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572599.486141:0:266400:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c372ecca780: ostid: 0x280000401:21134 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572599.486145:0:266400:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711cba000) owner ffff9c3711ea0000 00020000:00000001:0.0:1788572599.486149:0:266400:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572599.486153:0:266400:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711cba000) owner ffff9c3711ea0000 00020000:00000010:0.0:1788572599.486160:0:266400:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c372ecca780. 00020000:00000010:0.0:1788572599.486167:0:266400:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626c60. 00020000:00000010:0.0:1788572599.486172:0:266400:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626d80. 00020000:00000002:0.0:1788572599.486178:0:266400:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdfe:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572599.486183:0:266400:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.486189:0:266400:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572599.486193:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572599.486205:0:266400:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329150 : 144115373145329150 : 200002b17000dfe) 00000080:00000001:0.0:1788572599.486209:0:266400:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00200000:0.0:1788572599.486215:0:266400:0:(namei.c:147:ll_iget()) got inode: [0x200002b17:0xdfe:0x0](ffff9c37020e7448): rc = 0 00000080:00000001:0.0:1788572599.486221:0:266400:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358804477000 : -109714905074616 : ffff9c37020e7448) 00010000:00000001:0.0:1788572599.486225:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.486228:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.486231:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.486236:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486242:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00000080:00000001:0.0:1788572599.486249:0:266400:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572599.486251:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.486252:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572599.486256:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572599.486261:0:266400:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572599.486265:0:266400:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572599.486268:0:266400:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572599.486271:0:266400:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626d80. 00020000:00000010:0.0:1788572599.486276:0:266400:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c372ecca780. 00020000:00000010:0.0:1788572599.486282:0:266400:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626c60. 00020000:00000001:0.0:1788572599.486287:0:266400:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061638240 : -109714647913376 : ffff9c3711626c60) 00020000:00000002:0.0:1788572599.486292:0:266400:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626c60, objid 0xdfe:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572599.486296:0:266400:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572599.486303:0:266400:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c372ecca780: ostid: 0x280000401:21134 ost idx: 0 gen: 0 00020000:00000002:0.0:1788572599.486308:0:266400:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3711cba000) owner ffff9c3711ea0000 00020000:00000001:0.0:1788572599.486312:0:266400:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572599.486316:0:266400:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3711cba000) owner ffff9c3711ea0000 00020000:00000010:0.0:1788572599.486320:0:266400:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c372ecca780. 00020000:00000010:0.0:1788572599.486324:0:266400:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626d80. 00020000:00000010:0.0:1788572599.486327:0:266400:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626c60. 00020000:00000002:0.0:1788572599.486331:0:266400:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdfe:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572599.486333:0:266400:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.486336:0:266400:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.486346:0:266400:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572599.486350:0:266400:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00020000:00000002:0.0:1788572599.486353:0:266400:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3732215900 addref 2/0 by ffff9c3711ea0000. 00020000:00000002:0.0:1788572599.486357:0:266400:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00020000:00000001:0.0:1788572599.486361:0:266400:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572599.486365:0:266400:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572599.486369:0:266400:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572599.486371:0:266400:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572599.486374:0:266400:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572599.486379:0:266400:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdfe:0x0]: layout version change: 4294967294 -> 0 00000020:00001000:0.0:1788572599.486383:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572599.486387:0:266400:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.486391:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486394:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.486397:0:266400:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572599.486401:0:266400:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572599.486405:0:266400:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.486409:0:266400:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdfe:0x0](ffff9c37020e7448) for lock 0x144158609316897e 00800000:00000001:0.0:1788572599.486414:0:266400:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.486418:0:266400:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.486421:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.486424:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.486429:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.486433:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486438:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486441:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486442:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.486442:0:266400:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.486443:0:266400:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00002000:0.0:1788572599.486450:0:266400:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37034e2af8 inode ffff9c37020e7448 refc 1 flags 0x40000e 00000080:00000001:0.0:1788572599.486453:0:266400:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00010000:0.0:1788572599.486458:0:266400:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdfe:0x0](ffff9c37020e7448) 00000080:00010000:0.0:1788572599.486462:0:266400:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316897e from it ffffb95bc2073b20 00010000:00000001:0.0:1788572599.486465:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.486467:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.486476:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.486480:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486483:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486486:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.486489:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.486500:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486503:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.486506:0:266400:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.486509:0:266400:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.486512:0:266400:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.486522:0:266400:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.486525:0:266400:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.486527:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.486531:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486534:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.486537:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.486539:0:266400:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.486542:0:266400:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.486545:0:266400:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.486549:0:266400:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.486552:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.486553:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486554:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572599.486555:0:266400:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572599.486557:0:266400:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20000. 00000100:00000001:0.0:1788572599.486565:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.486570:0:266400:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5c00 x1875440809985920/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/640 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.486586:0:266400:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.486590:0:266400:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.486591:0:266400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572599.486601:0:266400:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.486604:0:266400:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.486607:0:266400:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.486609:0:266400:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.486612:0:266400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572599.486616:0:266400:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.486618:0:266400:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.486620:0:266400:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5c00. 00000100:00000001:0.0:1788572599.486623:0:266400:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.486626:0:266400:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.486630:0:266400:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572599.486632:0:266400:0:(dcache.c:176:ll_intent_release()) intent ffffb95bc2073b20 released 00000080:00000001:0.0:1788572599.486668:0:266400:0:(dcache.c:187:ll_intent_release()) Process leaving 00000020:00000001:0.0:1788572599.486680:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073d08 00000080:00200008:0.0:1788572599.486685:0:266400:0:(file.c:6296:ll_getattr_dentry()) START file f12:[0x200002b17:0xdfe:0x0](ffff9c37020e7448), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572599.486694:0:266400:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572599.486696:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073c80 00000080:00200000:0.0:1788572599.486699:0:266400:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200002b17:0xdfe:0x0](ffff9c37020e7448),name=f12 00000080:00000010:0.0:1788572599.486703:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.486709:0:266400:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.486712:0:266400:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.486715:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073ba0 00800000:00000002:0.0:1788572599.486719:0:266400:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200002b17:0xdfe:0x0] 'f12' on [0x200002b17:0xdfc:0x0] 00800000:00000001:0.0:1788572599.486722:0:266400:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.486724:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.486726:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.486728:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.486730:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.486733:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.486735:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfc:0x0] 00800000:00000001:0.0:1788572599.486738:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.486740:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.486742:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.486743:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.486746:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.486747:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.486748:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfe:0x0] 00800000:00000001:0.0:1788572599.486755:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.486758:0:266400:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200002b17:0xdfc:0x0], fid2=[0x200002b17:0xdfe:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572599.486762:0:266400:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.486765:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073a70 00000002:00010000:0.0:1788572599.486769:0:266400:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200002b17:0xdfe:0x0]) in obj [0x200002b17:0xdfc:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572599.486773:0:266400:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572599.486776:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.486778:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.486782:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 2 00010000:00000001:0.0:1788572599.486786:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.486790:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.486794:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572599.486800:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1788572599.486809:0:266400:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572599.486812:0:266400:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.486816:0:266400:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 2/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1788572599.486824:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572599.486827:0:266400:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572599.486830:0:266400:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.486835:0:266400:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.486838:0:266400:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.486842:0:266400:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b17:0xdfc:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1788572599.486847:0:266400:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:0.0:1788572599.486852:0:266400:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.486858:0:266400:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572599.486862:0:266400:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.486867:0:266400:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdfe:0x0](ffff9c37020e7448) 00000080:00010000:0.0:1788572599.486872:0:266400:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdfe:0x0](ffff9c37020e7448) for lock 0x144158609316897e 00800000:00000001:0.0:1788572599.486925:0:266400:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.486934:0:266400:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.486938:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.486941:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.486944:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.486949:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486954:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486958:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.486961:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.486963:0:266400:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.486967:0:266400:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.486972:0:266400:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x144158609316897e from it ffffb95bc2073cb8 00010000:00000001:0.0:1788572599.486976:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.486979:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.486984:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=3 00000020:00000001:0.0:1788572599.486987:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486990:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572599.486993:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.486996:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.487003:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.487005:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.487009:0:266400:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.487010:0:266400:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.487012:0:266400:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000001:0.0:1788572599.487015:0:266400:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.487020:0:266400:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.487022:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.487026:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.487028:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.487029:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487031:0:266400:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.487032:0:266400:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487041:0:266400:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487044:0:266400:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487049:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.487051:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.487054:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572599.487057:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572599.487060:0:266400:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.487064:0:266400:0:(pcc.c:2723:pcc_inode_getattr()) Process entered 00000080:00000001:0.0:1788572599.487067:0:266400:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572599.487069:0:266400:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572599.487074:0:266400:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572599.487075:0:266400:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.487078:0:266400:0:(pcc.c:2732:pcc_inode_getattr()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572599.487083:0:266400:0:(glimpse.c:178:__cl_glimpse_size()) Process entered 00000020:00000001:0.0:1788572599.487086:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.487088:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572599.487091:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572599.487096:0:266400:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebaa8 -> ffff9c3718eebaa8 00000020:00000001:0.0:1788572599.487100:0:266400:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572599.487104:0:266400:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572599.487108:0:266400:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdfe:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000020:00000001:0.0:1788572599.487112:0:266400:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572599.487115:0:266400:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572599.487117:0:266400:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572599.487121:0:266400:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.487126:0:266400:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572599.487131:0:266400:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdfe:0x0]io ffff9c370517ac50 type 7 ignore/verify layout 0/1 00020000:00000002:0.0:1788572599.487135:0:266400:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00020000:00000001:0.0:1788572599.487140:0:266400:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572599.487142:0:266400:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572599.487145:0:266400:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572599.487148:0:266400:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487151:0:266400:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572599.487155:0:266400:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572599.487158:0:266400:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487161:0:266400:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572599.487162:0:266400:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572599.487165:0:266400:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572599.487167:0:266400:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00000020:00000001:0.0:1788572599.487173:0:266400:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1788572599.487177:0:266400:0:(glimpse.c:70:cl_glimpse_lock()) Process entered 00000001:00010000:0.0:1788572599.487179:0:266400:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b17:0xdfe:0x0] 00000020:00000001:0.0:1788572599.487182:0:266400:0:(cl_lock.c:202:cl_lock_request()) Process entered 00000020:00000001:0.0:1788572599.487183:0:266400:0:(cl_lock.c:90:cl_lock_init()) Process entered 00020000:00000001:0.0:1788572599.487185:0:266400:0:(lov_lock.c:336:lov_lock_init_composite()) Process entered 00020000:00000001:0.0:1788572599.487186:0:266400:0:(lov_lock.c:110:lov_lock_sub_init()) Process entered 00020000:00000002:0.0:1788572599.487190:0:266400:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000010:0.0:1788572599.487196:0:266400:0:(lov_lock.c:147:lov_lock_sub_init()) kmalloced '(lovlck)': 112 at ffff9c372d676f00. 00020000:00000002:0.0:1788572599.487199:0:266400:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572599.487202:0:266400:0:(lov_lock.c:66:lov_sublock_init()) Process entered 00020000:00000001:0.0:1788572599.487205:0:266400:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572599.487207:0:266400:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572599.487209:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.487212:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359600763320 : -109714108788296 : ffff9c373184d1b8) 00000020:00001000:0.0:1788572599.487214:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c373184d1b0 00000020:00000001:0.0:1788572599.487218:0:266400:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebe90 -> ffff9c3718eebaa8 00000020:00000001:0.0:1788572599.487219:0:266400:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572599.487222:0:266400:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572599.487222:0:266400:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572599.487223:0:266400:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487225:0:266400:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487226:0:266400:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859894880 : -109714849656736 : ffff9c37055c1060) 00000020:00000001:0.0:1788572599.487227:0:266400:0:(cl_lock.c:90:cl_lock_init()) Process entered 00000008:00000010:0.0:1788572599.487229:0:266400:0:(osc_lock.c:1271:osc_lock_init()) slab-alloced 'oscl': 304 at ffff9c37030a6d10. 00000008:00010000:0.0:1788572599.487231:0:266400:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000020:00000001:0.0:1788572599.487233:0:266400:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572599.487233:0:266400:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00000008:00010000:0.0:1788572599.487234:0:266400:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9c372d676f30, osc lock ffff9c37030a6d10, flags 0x400001002 00000020:00000001:0.0:1788572599.487236:0:266400:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487237:0:266400:0:(lov_lock.c:76:lov_sublock_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487239:0:266400:0:(lov_lock.c:208:lov_lock_sub_init()) Process leaving (rc=18446634359531728640 : -109714177822976 : ffff9c372d676f00) 00000020:00000001:0.0:1788572599.487240:0:266400:0:(cl_lock.c:58:cl_lock_slice_add()) Process entered 00000020:00000001:0.0:1788572599.487240:0:266400:0:(cl_lock.c:63:cl_lock_slice_add()) Process leaving 00020000:00000001:0.0:1788572599.487241:0:266400:0:(lov_lock.c:342:lov_lock_init_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487243:0:266400:0:(cl_lock.c:106:cl_lock_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487245:0:266400:0:(cl_io.c:1189:cl_sync_io_init_notify()) Process entered 00000020:00000001:0.0:1788572599.487246:0:266400:0:(cl_io.c:1197:cl_sync_io_init_notify()) Process leaving 00000020:00000001:0.0:1788572599.487253:0:266400:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572599.487254:0:266400:0:(lov_lock.c:255:lov_lock_enqueue()) Process entered 00020000:00000001:0.0:1788572599.487256:0:266400:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572599.487257:0:266400:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859894880 : -109714849656736 : ffff9c37055c1060) 00000020:00000001:0.0:1788572599.487259:0:266400:0:(cl_lock.c:169:cl_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572599.487261:0:266400:0:(osc_lock.c:1005:osc_lock_enqueue()) Process entered 00000008:00000001:0.0:1788572599.487263:0:266400:0:(osc_lock.c:1030:osc_lock_enqueue()) Process leaving via enqueue_base (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788572599.487266:0:266400:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00020000:00000001:0.0:1788572599.487268:0:266400:0:(lovsub_object.c:148:lovsub_req_projid_set()) Process entered 00000020:00000001:0.0:1788572599.487270:0:266400:0:(cl_object.c:498:cl_req_projid_set()) Process entered 00000020:00000001:0.0:1788572599.487271:0:266400:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00020000:00000001:0.0:1788572599.487273:0:266400:0:(lovsub_object.c:150:lovsub_req_projid_set()) Process leaving 00000020:00000001:0.0:1788572599.487274:0:266400:0:(cl_object.c:504:cl_req_projid_set()) Process leaving 00000008:00000001:0.0:1788572599.487276:0:266400:0:(osc_request.c:3189:osc_enqueue_base()) Process entered 00010000:00000001:0.0:1788572599.487278:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572599.487281:0:266400:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487284:0:266400:0:(ldlm_request.c:996:ldlm_enqueue_pack()) Process entered 00000100:00000010:0.0:1788572599.487287:0:266400:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572599.487290:0:266400:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00010000:00000001:0.0:1788572599.487293:0:266400:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.487295:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.487298:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.487299:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.487300:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.487301:0:266400:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.487303:0:266400:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.487305:0:266400:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.487307:0:266400:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3711c20000. 00000100:00000001:0.0:1788572599.487310:0:266400:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487312:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.487313:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487314:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.487315:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487316:0:266400:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.487318:0:266400:0:(ldlm_request.c:1010:ldlm_enqueue_pack()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00010000:00000001:0.0:1788572599.487320:0:266400:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572599.487321:0:266400:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572599.487325:0:266400:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46a00. 00010000:00000010:0.0:1788572599.487331:0:266400:0:(ldlm_resource.c:1471:ldlm_resource_extent_new()) slab-alloced 'res->lr_itree': 216 at ffff9c3713c46c00. 00010000:00000001:0.0:1788572599.487334:0:266400:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572599.487336:0:266400:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30a00. 00000020:00000001:0.0:1788572599.487338:0:266400:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572599.487340:0:266400:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30a00 with handle 0x14415860931689a1 to hash 00000020:00000001:0.0:1788572599.487341:0:266400:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572599.487343:0:266400:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000010:0.0:1788572599.487347:0:266400:0:(ldlm_lock.c:1732:ldlm_lock_create()) kmalloced '(lock->l_lvb_data)': 56 at ffff9c372e9a9800. 00010000:00000001:0.0:1788572599.487350:0:266400:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.487352:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.487354:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.487357:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572599.487366:0:266400:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.487377:0:266400:0:(ldlm_request.c:1191:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.487381:0:266400:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.487383:0:266400:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.487390:0:266400:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000008:00000001:0.0:1788572599.487418:0:266400:0:(osc_request.c:3308:osc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572599.487421:0:266400:0:(osc_lock.c:1109:osc_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487423:0:266400:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.487425:0:266400:0:(lov_lock.c:274:lov_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487426:0:266400:0:(cl_lock.c:179:cl_lock_enqueue()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.487429:0:266400:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572599.487431:0:266400:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00000001:0.0:1788572599.487433:0:266400:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000020:00000001:0.0:1788572599.487435:0:266400:0:(cl_io.c:1209:cl_sync_io_wait()) Process entered 00000100:00000001:0.0:1788572599.487442:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572599.487451:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572599.487676:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572599.487680:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572599.487684:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572599.487689:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572599.487691:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572599.487692:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572599.487695:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572599.487698:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.487703:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.487704:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.487709:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.487710:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572599.490725:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.490730:0:266396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.490734:0:266396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.490738:0:266396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37050c5160. 00000400:00000200:0.0:1788572599.490744:0:266396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1f68 00000400:00000010:0.0:1788572599.490747:0:266396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1f68. 00000100:00000001:0.0:1788572599.490751:0:266396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.490760:0:266396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572599.490777:0:266396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.490800:0:266396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.490805:0:266396:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.490808:0:266396:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.490812:0:266396:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.490825:0:266396:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.490833:0:266396:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.490837:0:266396:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.490844:0:266396:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=13 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572599.490852:0:266396:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=74 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572599.490863:0:266396:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1788572599.490872:0:266396:0:(client.c:1517:ptlrpc_check_status()) @@@ check status: rc = -115 req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/200/0 rc 0/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.490891:0:266396:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000001:0.0:1788572599.490894:0:266396:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.490896:0:266396:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869953 00000100:00000001:0.0:1788572599.490898:0:266396:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.490899:0:266396:0:(client.c:1805:after_reply()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000040:0.0:1788572599.490905:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.490911:0:266396:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551501 : -115 : 0xffffffffffffff8d) 00000100:00000001:0.0:1788572599.490914:0:266396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.490915:0:266396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572599.490920:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572684 ref 2 fl Interpret:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572599.490928:0:266396:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266396:1875440809986304:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572599.490933:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.490935:0:266396:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000001:0.0:1788572599.490937:0:266396:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572599.490938:0:266396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.490941:0:266396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572684 ref 2 fl Complete:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.490947:0:266396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.490950:0:266396:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c372dbfcf00. 00000100:00000001:0.0:1788572599.490953:0:266396:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572599.490954:0:266396:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000002:00000040:0.0:1788572599.490958:0:266396:0:(mdc_reint.c:34:mdc_reint()) error in handling -115 00000100:00000001:0.0:1788572599.490960:0:266396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.490963:0:266396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440809986304/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572684 ref 1 fl Complete:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.490980:0:266396:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.490982:0:266396:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.490985:0:266396:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371ff2d000. 02000000:00000001:0.0:1788572599.490989:0:266396:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.490992:0:266396:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.490996:0:266396:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572599.491000:0:266396:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.491004:0:266396:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c370395b000. 02000000:00000001:0.0:1788572599.491017:0:266396:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.491020:0:266396:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.491024:0:266396:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788572599.491028:0:266396:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.491029:0:266396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000002:00080000:0.0:1788572599.491035:0:266396:0:(mdc_reint.c:252:mdc_create()) lustre-MDT0001-mdc-ffff9c3711155800: resend:1 create on [0x2400013a6:0xd83:0x0]/[0x2400013a6:0xd87:0x0] 00000002:00000001:0.0:1788572599.491045:0:266396:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572599.491053:0:266396:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3704ff6400 count: 2 00010000:00000001:0.0:1788572599.491060:0:266396:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.491065:0:266396:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572599.491069:0:266396:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491072:0:266396:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.491077:0:266396:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572599.491085:0:266396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572599.491091:0:266396:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572599.491100:0:266396:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572599.491105:0:266396:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491110:0:266396:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.491114:0:266396:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.491120:0:266396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.491122:0:266396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.491127:0:266396:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491132:0:266396:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.491136:0:266396:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.491142:0:266396:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.491152:0:266396:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3711c21c00. 00000100:00000001:0.0:1788572599.491167:0:266396:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491174:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.491179:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491184:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.491188:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491195:0:266396:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000020:00100000:0.0:1788572599.491209:0:266396:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788572599.491229:0:266396:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b4e00 x1875440809987200/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572599.491247:0:266396:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.491251:0:266396:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.491259:0:266396:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a780. 00000100:00000001:0.0:1788572599.491264:0:266396:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040026496 : -109714669525120 : ffff9c371018a780) 00000100:00000001:0.0:1788572599.491271:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.491274:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788572599.491281:0:266396:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.491287:0:266396:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491292:0:266396:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.491296:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572599.491300:0:266396:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491306:0:266396:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.491311:0:266396:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572599.491317:0:266396:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a780 going to sleep for 1 seconds 00000100:00000001:0.0:1788572599.491329:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.491335:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572599.491339:0:266396:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491345:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491455:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572599.491461:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.491478:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.491481:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.491486:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2268. 00000400:00000200:0.0:1788572599.491493:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788572599.491498:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788572599.491504:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.491519:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572599.491546:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.491562:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.491567:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.491571:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.491575:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.491578:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.491581:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.491585:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572599.491594:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=13 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572599.491602:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=64 v=5 (1 1 2 0) 00000100:00000001:0.0:1788572599.491609:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572599.491613:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491618:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572599.491622:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.491627:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572599.491630:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727881 00000100:00000001:0.0:1788572599.491661:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572599.491666:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.491676:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Rpc:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.491703:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.491709:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572599.491714:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788572599.491720:0:2403:0:(osc_request.c:3123:osc_enqueue_interpret()) Process entered 00010000:00000001:0.0:1788572599.491724:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.491728:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.491731:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=4 00000020:00000001:0.0:1788572599.491736:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.491741:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.491746:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.491751:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.491755:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788572599.491766:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.491773:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.491779:0:2403:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.491785:0:2403:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.491794:0:2403:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.491810:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.491813:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.491818:0:2403:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572599.491824:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.491827:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.491832:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=6 00000020:00000001:0.0:1788572599.491837:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.491843:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000040:0.0:1788572599.491849:0:2403:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c372db30a00, remote cookie: 0xd9894672d99a30d0, flags: 0x1001 00010000:00010000:0.0:1788572599.491858:0:2403:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.491892:0:2403:0:(ldlm_lock.c:1589:ldlm_fill_lvb()) Process entered 00010000:00000001:0.0:1788572599.491898:0:2403:0:(ldlm_lock.c:1682:ldlm_fill_lvb()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.491904:0:2403:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572599.491909:0:2403:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00010000:0.0:1788572599.491915:0:2403:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572599.491931:0:2403:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x280000401:0x528e:0x0].0x0 (ffff9c3713c46a00) refcount = 1 00010000:00000040:0.0:1788572599.491938:0:2403:0:(ldlm_resource.c:1879:ldlm_resource_dump()) Granted locks (in reverse order): 00010000:00000040:0.0:1788572599.491944:0:2403:0:(ldlm_resource.c:1882:ldlm_resource_dump()) ### ### ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.491997:0:2403:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572599.492001:0:2403:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.492008:0:2403:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572599.492012:0:2403:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.492020:0:2403:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.492035:0:2403:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572599.492039:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492043:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.492046:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492049:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572599.492053:0:2403:0:(osc_request.c:3077:osc_enqueue_fini()) Process entered 00000008:00000001:0.0:1788572599.492058:0:2403:0:(osc_lock.c:272:osc_lock_upcall()) Process entered 00010000:00000001:0.0:1788572599.492064:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.492067:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.492071:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788572599.492076:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.492081:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00000008:00000001:0.0:1788572599.492089:0:2403:0:(osc_lock.c:149:osc_lock_lvb_update()) Process entered 00000020:00000001:0.0:1788572599.492095:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730f60c40 -> ffff9c3718eebaa8 00000008:00010000:0.0:1788572599.492102:0:2403:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1788572599.492124:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730f60c40 -> ffff9c3718eebaa8 00000020:00000001:0.0:1788572599.492130:0:2403:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572599.492134:0:2403:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572599.492140:0:2403:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572599.492145:0:2403:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492150:0:2403:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492156:0:2403:0:(cl_object.c:143:cl_object_top()) ffff9c3730f60c40 -> ffff9c3718eebaa8 00000008:00000001:0.0:1788572599.492167:0:2403:0:(osc_lock.c:192:osc_lock_lvb_update()) Process leaving 00000020:00000001:0.0:1788572599.492172:0:2403:0:(cl_io.c:1833:__cl_sync_io_note()) Process entered 00000020:00200000:0.0:1788572599.492176:0:2403:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00000001:0.0:1788572599.492188:0:2403:0:(cl_io.c:1907:__cl_sync_io_note()) Process leaving 00000008:00000001:0.0:1788572599.492192:0:2403:0:(osc_lock.c:306:osc_lock_upcall()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492198:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.492202:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.492206:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=6 00000020:00000001:0.0:1788572599.492211:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.492215:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.492221:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.492226:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.492246:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492250:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572599.492254:0:2403:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.492267:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.492271:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492275:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572599.492278:0:2403:0:(osc_request.c:3104:osc_enqueue_fini()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492283:0:2403:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.492287:0:2403:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.492291:0:2403:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788572599.492298:0:2403:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.492306:0:2403:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788572599.492311:0:2403:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.492317:0:2403:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572599.492337:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492346:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.492350:0:2403:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.492355:0:2403:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.492362:0:2403:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000a40bf179/0x14415860931689a1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x528e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0xd9894672d99a30d0 expref: -99 pid: 266400 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572599.492380:0:2403:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.492384:0:2403:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.492388:0:2403:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.492394:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492398:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.492402:0:2403:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492408:0:2403:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.492412:0:2403:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492418:0:2403:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492423:0:2403:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.492428:0:2403:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.492432:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492436:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.492439:0:2403:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492442:0:2403:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572599.492446:0:2403:0:(osc_request.c:3162:osc_enqueue_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.492455:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Interpret:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572599.492473:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809986944:192.168.202.148@tcp:101:rm.0 00000100:00000001:0.0:1788572599.492483:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.492486:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.492493:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5c00 x1875440809986944/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.492509:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572599.492513:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572599.492517:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c373233dc00. 02000000:00000001:0.0:1788572599.492524:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572599.492528:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572599.492533:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572599.492543:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572599.492548:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711c20000. 02000000:00000001:0.0:1788572599.492557:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572599.492560:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572599.492565:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5c00. 00000100:00000001:0.0:1788572599.492572:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572599.492576:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.492581:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.492590:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.492594:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.492599:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572599.492602:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492613:0:266400:0:(cl_io.c:1233:cl_sync_io_wait()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492619:0:266400:0:(cl_lock.c:230:cl_lock_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.492625:0:266400:0:(file.c:1724:ll_merge_attr_nolock()) Process entered 00000020:00000001:0.0:1788572599.492630:0:266400:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebaa8 -> ffff9c3718eebaa8 00000020:00000001:0.0:1788572599.492658:0:266400:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebaa8 -> ffff9c3718eebaa8 00000020:00000001:0.0:1788572599.492662:0:266400:0:(cl_object.c:210:cl_object_attr_get()) Process entered 00020000:00000001:0.0:1788572599.492666:0:266400:0:(lov_object.c:1069:lov_attr_get_composite()) Process entered 00020000:00000001:0.0:1788572599.492671:0:266400:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572599.492676:0:266400:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572599.492681:0:266400:0:(lov_offset.c:41:lov_stripe_size()) Process entered 00020000:00000001:0.0:1788572599.492684:0:266400:0:(lov_offset.c:44:lov_stripe_size()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572599.492688:0:266400:0:(lov_merge.c:82:lov_merge_lvb_kms()) MDT ID 0xdfe:11031 on OST[0]: s=0 (1) m=1788572598 a=1788572598 c=1788572598 b=0 00020000:00000001:0.0:1788572599.492699:0:266400:0:(lov_merge.c:93:lov_merge_lvb_kms()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572599.492705:0:266400:0:(lov_object.c:1100:lov_attr_get_composite()) COMP ID #-1: s=0 m=1788572598 a=1788572598 c=1788572598 b=0 00020000:00000001:0.0:1788572599.492710:0:266400:0:(lov_object.c:1138:lov_attr_get_composite()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492716:0:266400:0:(cl_object.c:222:cl_object_attr_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492722:0:266400:0:(cl_object.c:143:cl_object_top()) ffff9c3718eebaa8 -> ffff9c3718eebaa8 00000080:00200000:0.0:1788572599.492729:0:266400:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b17:0xdfe:0x0] updating i_size 0 i_blocks 0 00000080:00000001:0.0:1788572599.492737:0:266400:0:(file.c:1794:ll_merge_attr_nolock()) Process leaving 00000020:00000001:0.0:1788572599.492742:0:266400:0:(cl_lock.c:243:cl_lock_release()) Process entered 00000020:00010000:0.0:1788572599.492746:0:266400:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9c370517abf0 (ffff9c3711a1dab8/0) at cl_lock_release():245 00000020:00000001:0.0:1788572599.492753:0:266400:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572599.492757:0:266400:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c370517abf0 (ffff9c3711a1dab8/0) at cl_lock_cancel():142 00020000:00000001:0.0:1788572599.492765:0:266400:0:(lov_lock.c:284:lov_lock_cancel()) Process entered 00020000:00000001:0.0:1788572599.492777:0:266400:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572599.492780:0:266400:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634358859894880 : -109714849656736 : ffff9c37055c1060) 00000020:00000001:0.0:1788572599.492785:0:266400:0:(cl_lock.c:140:cl_lock_cancel()) Process entered 00000020:00010000:0.0:1788572599.492788:0:266400:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9c372d676f30 (ffff9c373184d1b8/1) at cl_lock_cancel():142 00000008:00000001:0.0:1788572599.492795:0:266400:0:(osc_lock.c:1167:osc_lock_cancel()) Process entered 00000008:00000001:0.0:1788572599.492799:0:266400:0:(osc_lock.c:1121:osc_lock_detach()) Process entered 00010000:00000001:0.0:1788572599.492803:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.492807:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000008:00000001:0.0:1788572599.492810:0:266400:0:(osc_lock.c:1140:osc_lock_detach()) Process leaving 00000008:00000001:0.0:1788572599.492814:0:266400:0:(osc_lock.c:1176:osc_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572599.492818:0:266400:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572599.492821:0:266400:0:(cl_lock.c:148:cl_lock_cancel()) Process leaving 00000020:00000001:0.0:1788572599.492825:0:266400:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572599.492829:0:266400:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c370517abf0 (ffff9c3711a1dab8/0) at cl_lock_fini():72 00020000:00000001:0.0:1788572599.492835:0:266400:0:(lov_lock.c:217:lov_lock_fini()) Process entered 00000020:00000001:0.0:1788572599.492838:0:266400:0:(cl_lock.c:71:cl_lock_fini()) Process entered 00000020:00010000:0.0:1788572599.492841:0:266400:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9c372d676f30 (ffff9c3711a1dab8/1) at cl_lock_fini():72 00000008:00000010:0.0:1788572599.492849:0:266400:0:(osc_lock.c:111:osc_lock_fini()) slab-freed '(ols)': 304 at ffff9c37030a6d10. 00000020:00000001:0.0:1788572599.492856:0:266400:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00020000:00000010:0.0:1788572599.492859:0:266400:0:(lov_lock.c:225:lov_lock_fini()) kvfree 'lovlck': 112 at ffff9c372d676f00. 00020000:00000001:0.0:1788572599.492869:0:266400:0:(lov_lock.c:226:lov_lock_fini()) Process leaving 00000020:00000001:0.0:1788572599.492873:0:266400:0:(cl_lock.c:80:cl_lock_fini()) Process leaving 00000020:00000001:0.0:1788572599.492888:0:266400:0:(cl_lock.c:248:cl_lock_release()) Process leaving 00000001:00000001:0.0:1788572599.492890:0:266400:0:(glimpse.c:118:cl_glimpse_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492896:0:266400:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572599.492901:0:266400:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572599.492904:0:266400:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572599.492908:0:266400:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572599.492912:0:266400:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572599.492915:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c373184d1b0 00020000:00000001:0.0:1788572599.492919:0:266400:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572599.492923:0:266400:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572599.492927:0:266400:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572599.492932:0:266400:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdfe:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572599.492941:0:266400:0:(file.c:7109:ll_layout_refresh()) Process entered 00000080:00000001:0.0:1788572599.492944:0:266400:0:(file.c:7113:ll_layout_refresh()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.492949:0:266400:0:(vvp_io.c:399:vvp_io_fini()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572599.492955:0:266400:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572599.492958:0:266400:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572599.492961:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000001:00000001:0.0:1788572599.492972:0:266400:0:(glimpse.c:211:__cl_glimpse_size()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.492982:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073d08 00000080:00000008:0.0:1788572599.492989:0:266400:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file f12:[0x200002b17:0xdfe:0x0](ffff9c37020e7448), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1788572599.492998:0:266400:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572599.493002:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073da0 00000080:00002000:0.0:1788572599.493008:0:266400:0:(dcache.c:127:ll_ddelete()) keeping dentry f12 (ffff9c37034e2af8, parent ffff9c37034e2000, inode ffff9c37020e7448) hashed, 00000080:00000001:0.0:1788572599.493017:0:266400:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493043:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.493049:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:0.0:1788572599.493053:0:266400:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1788572599.493057:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073bd8 00000080:00200000:0.0:1788572599.493063:0:266400:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08),name=/ 00000080:00000010:0.0:1788572599.493073:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00800000:00000002:0.0:1788572599.493080:0:266400:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff9c37184144c0 2 00000080:00000001:0.0:1788572599.493087:0:266400:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:0.0:1788572599.493091:0:266400:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.493096:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073af8 00800000:00000002:0.0:1788572599.493101:0:266400:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.493110:0:266400:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1788572599.493114:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.493118:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.493122:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.493127:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.493132:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493136:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1788572599.493141:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493145:0:266400:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1788572599.493151:0:266400:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1788572599.493154:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc20739c8 00000002:00010000:0.0:1788572599.493158:0:266400:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1788572599.493166:0:266400:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1788572599.493169:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.493171:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.493175:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46800 count: 2 00010000:00000001:0.0:1788572599.493179:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.493185:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.493196:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788572599.493203:0:266400:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 02000000:00000001:0.0:1788572599.493220:0:266400:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1788572599.493225:0:266400:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572599.493232:0:266400:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000002:00000001:0.0:1788572599.493245:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1788572599.493249:0:266400:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572599.493253:0:266400:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.493257:0:266400:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1788572599.493261:0:266400:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.493265:0:266400:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00000001:0.0:1788572599.493271:0:266400:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1788572599.493275:0:266400:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff9c37184144c0 2 00000080:00000010:0.0:1788572599.493279:0:266400:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20000. 00000080:00000001:0.0:1788572599.493286:0:266400:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1788572599.493290:0:266400:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.493295:0:266400:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) 00000080:00010000:0.0:1788572599.493298:0:266400:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9c372dcc6c08) for lock 0x1441586093168923 00800000:00000001:0.0:1788572599.493301:0:266400:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572599.493303:0:266400:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572599.493304:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.493305:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.493307:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=3 00000020:00000001:0.0:1788572599.493309:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.493312:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.493314:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.493316:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572599.493317:0:266400:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.493319:0:266400:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572599.493321:0:266400:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x1441586093168923 from it ffffb95bc2073c10 00010000:00000001:0.0:1788572599.493333:0:266400:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572599.493335:0:266400:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572599.493336:0:266400:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30e00 refcount=3 00000020:00000001:0.0:1788572599.493338:0:266400:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.493341:0:266400:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684544 : -109714172867072 : ffff9c372db30e00) 00010000:00000001:0.0:1788572599.493343:0:266400:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572599.493346:0:266400:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00000001:0.0:1788572599.493353:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.493365:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.493368:0:266400:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572599.493372:0:266400:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572599.493381:0:266400:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30e00/0x1441586093168923 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a2f3a expref: -99 pid: 266396 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00000001:0.0:1788572599.493396:0:266400:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572599.493399:0:266400:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572599.493402:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.493407:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.493410:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.493413:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.493418:0:266400:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572599.493420:0:266400:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.493424:0:266400:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.493430:0:266400:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.493433:0:266400:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572599.493434:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.493436:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1788572599.493438:0:266400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788572599.493439:0:266400:0:(client.c:2877:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1788572599.493442:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9c372dcc6c08), inode mode 41ff mask 1 00000080:00000001:0.0:1788572599.493447:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493452:0:266400:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572599.493456:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073ba0 00000080:00002000:0.0:1788572599.493460:0:266400:0:(dcache.c:87:ll_dcompare()) found name d801b.sanity(ffff9c37034e2510) flags 0x20000e refc 6 00000080:00000001:0.0:1788572599.493472:0:266400:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.493477:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073be8 00000080:00200000:0.0:1788572599.493481:0:266400:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d801b.sanity, flags=24 00000080:00000001:0.0:1788572599.493484:0:266400:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.493488:0:266400:0:(llite_lib.c:3161:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493494:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.493498:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdf9:0x0](ffff9c3708704b08), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.493505:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493510:0:266400:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572599.493513:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073ba0 00000080:00002000:0.0:1788572599.493517:0:266400:0:(dcache.c:87:ll_dcompare()) found name d4(ffff9c37034e2000) flags 0x20000e refc 1 00000080:00000001:0.0:1788572599.493522:0:266400:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.493526:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073be8 00000080:00200000:0.0:1788572599.493529:0:266400:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d4, flags=24 00000080:00000001:0.0:1788572599.493534:0:266400:0:(llite_lib.c:3155:ll_dir_default_lmv_inherit()) Process entered 00000080:00000001:0.0:1788572599.493538:0:266400:0:(llite_lib.c:3165:ll_dir_default_lmv_inherit()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493543:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.493545:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8), inode mode 41ed mask 1 00000080:00000001:0.0:1788572599.493551:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493558:0:266400:0:(dcache.c:78:ll_dcompare()) Process entered 00000020:00000001:0.0:1788572599.493562:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073da0 00000080:00002000:0.0:1788572599.493566:0:266400:0:(dcache.c:87:ll_dcompare()) found name f12(ffff9c37034e2af8) flags 0x48000e refc 0 00000080:00000001:0.0:1788572599.493570:0:266400:0:(dcache.c:100:ll_dcompare()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.493575:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073e10 00000080:00200000:0.0:1788572599.493579:0:266400:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f12, flags=0 00000080:00000001:0.0:1788572599.493586:0:266400:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572599.493591:0:266400:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8), inode mode 41ed mask 3 00000080:00000001:0.0:1788572599.493596:0:266400:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.493603:0:266400:0:(namei.c:2348:ll_unlink()) Process entered 00000020:00000001:0.0:1788572599.493606:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073df8 00000080:00200000:0.0:1788572599.493610:0:266400:0:(namei.c:2351:ll_unlink()) VFS Op:name=f12, dir=[0x200002b17:0xdfc:0x0](ffff9c37081f80c8) 00000080:00000001:0.0:1788572599.493615:0:266400:0:(llite_foreign.c:180:ll_foreign_is_removable()) Process entered 00000020:00000001:0.0:1788572599.493618:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.493622:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572599.493628:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572599.493633:0:266400:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572599.493664:0:266400:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00020000:00000002:0.0:1788572599.493670:0:266400:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3732215900 addref 2/0 by ffff9c3711ea0000. 00020000:00000002:0.0:1788572599.493676:0:266400:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3711cba000) owner 0000000000000000/ffff9c3711ea0000 00020000:00000001:0.0:1788572599.493681:0:266400:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572599.493684:0:266400:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572599.493687:0:266400:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572599.493691:0:266400:0:(lov_pack.c:107:lov_lsm_pack_v1v3()) Process leaving (rc=18446744073709551582 : -34 : ffffffffffffffde) 00020000:00000001:0.0:1788572599.493697:0:266400:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=18446744073709551582 : -34 : ffffffffffffffde) 00000020:00001000:0.0:1788572599.493703:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572599.493708:0:266400:0:(llite_foreign.c:262:ll_foreign_is_removable()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:0.0:1788572599.493716:0:266400:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff9c3711c20000. 00800000:00000001:0.0:1788572599.493725:0:266400:0:(lmv_obd.c:3494:lmv_unlink()) Process entered 00800000:00000001:0.0:1788572599.493729:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.493732:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.493736:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.493740:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.493744:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493749:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfc:0x0] 00800000:00000001:0.0:1788572599.493755:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.493760:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.493764:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.493767:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.493771:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.493777:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493782:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfe:0x0] 00800000:00000001:0.0:1788572599.493787:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572599.493793:0:266400:0:(lmv_obd.c:2563:lmv_early_cancel()) Process entered 00800000:00000001:0.0:1788572599.493796:0:266400:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572599.493799:0:266400:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572599.493802:0:266400:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572599.493807:0:266400:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572599.493810:0:266400:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493815:0:266400:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdfe:0x0] 00800000:00000001:0.0:1788572599.493820:0:266400:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493824:0:266400:0:(lmv_obd.c:2582:lmv_early_cancel()) EARLY_CANCEL skip operation target 0 on [0x200002b17:0xdfe:0x0] 00800000:00000001:0.0:1788572599.493829:0:266400:0:(lmv_obd.c:2587:lmv_early_cancel()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572599.493840:0:266400:0:(lmv_obd.c:3534:lmv_unlink()) unlink with fid=[0x200002b17:0xdfc:0x0]/[0x200002b17:0xdfe:0x0] -> mds #0 00000002:00000001:0.0:1788572599.493849:0:266400:0:(mdc_reint.c:315:mdc_unlink()) Process entered 00000002:00000001:0.0:1788572599.493853:0:266400:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572599.493859:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46f00 count: 2 00010000:00000001:0.0:1788572599.493865:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.493869:0:266400:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572599.493871:0:266400:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.493874:0:266400:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.494043:0:266400:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.494049:0:266400:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572599.494054:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 2 00010000:00000001:0.0:1788572599.494060:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572599.494065:0:266400:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572599.494070:0:266400:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572599.494075:0:266400:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:0.0:1788572599.494090:0:266400:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572599.494093:0:266400:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572599.494096:0:266400:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572599.494099:0:266400:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xdfe:0x0] mode (null) 00000002:00000001:0.0:1788572599.494104:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.494107:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.494111:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572599.494115:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572599.494120:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.494125:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.494128:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.494133:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572599.494137:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572599.494143:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.494148:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.494151:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.494155:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572599.494159:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572599.494172:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572599.494176:0:266400:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572599.494179:0:266400:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572599.494184:0:266400:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572599.494188:0:266400:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572599.494193:0:266400:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.494197:0:266400:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.494201:0:266400:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572599.494204:0:266400:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572599.494208:0:266400:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572599.494213:0:266400:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572599.494217:0:266400:0:(cl_object.c:343:cl_conf_set()) Process entered 00000080:00200000:0.0:1788572599.494221:0:266400:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b17:0xdfe:0x0]: losing layout lock 00000080:00000001:0.0:1788572599.494224:0:266400:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered 00000080:00000001:0.0:1788572599.494229:0:266400:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving 00020000:00000001:0.0:1788572599.494231:0:266400:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572599.494235:0:266400:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572599.494241:0:266400:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdfe:0x0] lo_layout_invalid=1 00020000:00000001:0.0:1788572599.494245:0:266400:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572599.494249:0:266400:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572599.494253:0:266400:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572599.494258:0:266400:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572599.494262:0:266400:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788572599.494264:0:266400:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b17:0xdfe:0x0](ffff9c37020e7448) invalid 00000020:00000001:0.0:1788572599.494267:0:266400:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc2073a60 00000080:00002000:0.0:1788572599.494271:0:266400:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry f12 (ffff9c37034e2af8) parent ffff9c37034e2000 inode ffff9c37020e7448 refc 1 00000080:00000001:0.0:1788572599.494278:0:266400:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788572599.494282:0:266400:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572599.494286:0:266400:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.494293:0:266400:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572599.494298:0:266400:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572599.494301:0:266400:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572599.494305:0:266400:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572599.494310:0:266400:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788572599.494317:0:266400:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30400 with handle 0x144158609316897e from hash 00010000:00000001:0.0:1788572599.494324:0:266400:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572599.494333:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.494337:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.494340:0:266400:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572599.494344:0:266400:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572599.494347:0:266400:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572599.494352:0:266400:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572599.494356:0:266400:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572599.494361:0:266400:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788572599.494377:0:266400:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572599.494383:0:266400:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=6 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572599.494390:0:266400:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572599.494393:0:266400:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.494398:0:266400:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572599.494401:0:266400:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572599.494406:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572599.494409:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.494410:0:266400:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.494413:0:266400:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572599.494415:0:266400:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572599.494417:0:266400:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572599.494423:0:266400:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3711c20200. 00000100:00000001:0.0:1788572599.494429:0:266400:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.494443:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.494445:0:266400:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572599.494449:0:266400:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788572599.494459:0:266400:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572599.494462:0:266400:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572599.494466:0:266400:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572599.494470:0:266400:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c372db30400/0x144158609316897e lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xdfe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xd9894672d99a308a expref: -99 pid: 266400 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:0.0:1788572599.494485:0:266400:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372e9a9b40. 00010000:00000040:0.0:1788572599.494493:0:266400:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46b00 count: 0 00010000:00000010:0.0:1788572599.494507:0:266400:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff9c372d676300. 00010000:00000001:0.0:1788572599.494518:0:266400:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572599.494521:0:266400:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.494526:0:266400:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572599.494529:0:266400:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572599.494533:0:266400:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000020:00100000:0.0:1788572599.494545:0:266400:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1788572599.494556:0:266400:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b5c00 x1875440809987712/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/888 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572599.494570:0:266400:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572599.494573:0:266400:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572599.494580:0:266400:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a0c0. 00000100:00000001:0.0:1788572599.494585:0:266400:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040024768 : -109714669526848 : ffff9c371018a0c0) 00000100:00000001:0.0:1788572599.494591:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.494594:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000001:0.0:1788572599.494600:0:266400:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572599.494605:0:266400:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.494609:0:266400:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572599.494611:0:266400:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572599.494619:0:266400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5c00 x1875440809987712/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/888 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.494633:0:266400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572599.494674:0:266400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.494680:0:266400:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572599.494684:0:266400:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572599.494690:0:266400:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.494698:0:266400:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5c00 pname:cluuid:pid:xid:nid:opc:job rm:5fa795d2-07a0-4260-8bbc-bb3cde572406:266400:1875440809987712:192.168.202.148@tcp:36:rm.0 00000100:00000001:0.0:1788572599.494707:0:266400:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572599.494711:0:266400:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572599.494714:0:266400:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.494718:0:266400:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572599.494724:0:266400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572599.494729:0:266400:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572599.494746:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572599.494754:0:266400:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8840. 00000400:00000010:0.0:1788572599.494762:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572599.494769:0:266400:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809987712, portal 10 00000100:00000001:0.0:1788572599.494773:0:266400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572599.494775:0:266400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531985920 : -109714177565696 : ffff9c372d6b5c00) 00000100:00000040:0.0:1788572599.494786:0:266400:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b5c00 x1875440809987712/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/888 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.494800:0:266400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572599.494807:0:266400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572599.494811:0:266400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788572599.494816:0:266400:0:(niobuf.c:94:ptl_send_buf()) Sending 488 bytes to portal 12, xid 1875440809987712, offset 0 00000400:00000200:0.0:1788572599.494824:0:266400:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572599.494833:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.494840:0:266400:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572599.494846:0:266400:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572599.494852:0:266400:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572599.494860:0:266400:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572599.494866:0:266400:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572599.494870:0:266400:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540066:540066:255:4294967295] 192.168.202.148@tcp LPNI seq info [540031:540031:7:4294967295] 00000400:00000200:0.0:1788572599.494891:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.494902:0:266400:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572599.494907:0:266400:0:(socklnd_cb.c:931:ksocknal_send()) sending 488 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572599.494913:0:266400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788572599.494917:0:266400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572599.494922:0:266400:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572599.494925:0:266400:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0000 type 1, nob 584 niov 1 nkiov 1 00000100:00000001:0.0:1788572599.494929:0:266400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572599.494932:0:266400:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15057 usecs (16 jiffies) 00000400:00000200:0.0:1788572599.494935:0:266400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572599.494937:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307738517 00000100:00000001:0.0:1788572599.494946:0:266400:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307738662 00000100:00000001:0.0:1788572599.494950:0:266400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307738517 00000100:00000001:0.0:1788572599.494952:0:266400:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307738662 00000100:00000001:0.0:1788572599.494957:0:266400:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572599.494962:0:266400:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.494965:0:266400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.494968:0:266400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572599.494973:0:266400:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a0c0 going to sleep for 80 seconds 00000100:00000001:0.0:1788572599.494978:0:266400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.494981:0:266400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572599.496235:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.496246:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572599.496249:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.496252:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572599.496258:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.496265:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572599.496272:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.496276:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.496280:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.496284:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 584 into portal 10 MB=0x6a9b41746cd00 00000400:00000200:0.0:1788572599.496289:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.496295:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 584/584 into md 0x52aa81 [1] + 224 00000800:00000001:0.0:1788572599.496302:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.496309:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572599.496312:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572599.496317:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572599.496331:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.496341:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed68e00 x1875440809987328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572599.496352:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=584 offset=224 replen=1680 req@ffff9c371ed68e00 x1875440809987328/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.496386:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572599.496394:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.496398:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572599.497101:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.497109:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572599.497111:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.497113:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572599.497117:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.497122:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572599.497128:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.497133:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.497137:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.497142:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 4 MB=0x6a9b41746cd80 00000400:00000200:0.0:1788572599.497152:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572599.497159:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 400/400 into md 0x52aa89 [1] + 224 00000800:00000001:0.0:1788572599.497165:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.497204:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572599.497208:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572599.497215:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572599.497220:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.497224:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c6300 x1875440809987456/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572599.497238:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=400 req@ffff9c371f1c6300 x1875440809987456/t0(0) o101->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 328/400 e 0 to 0 dl 1788572615 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.497268:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572599.497274:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572599.497278:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572599.499386:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572599.499393:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5000. 00000400:00000200:0.0:1788572599.499399:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572599.499407:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572599.499420:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1220 00000400:00000010:0.0:1788572599.499423:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1220. 00000100:00000001:0.0:1788572599.499426:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572599.499430:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68e00 x1875440809987840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.499445:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572599.499450:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68e00 x1875440809987840/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/1680 e 0 to 0 dl 1788572684 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572599.499460:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.499465:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572599.504621:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572599.504626:0:266398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572599.504627:0:266398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572599.504629:0:266398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37050c5c60. 00000400:00000200:0.0:1788572599.504631:0:266398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1330 00000400:00000010:0.0:1788572599.504633:0:266398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1330. 00000100:00000001:0.0:1788572599.504662:0:266398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572599.504669:0:266398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71880 x1875440809988096/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572599.504683:0:266398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71880 x1875440809988096/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.504695:0:266398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572599.504698:0:266398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.504702:0:266398:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572599.504705:0:266398:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572599.504708:0:266398:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572599.504710:0:266398:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572599.504713:0:266398:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572599.504718:0:266398:0:(client.c:1659:after_reply()) @@@ resending request on EINPROGRESS req@ffff9c371ec71880 x1875440809988096/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RQU/200/0 rc 0/-115 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572599.504732:0:266398:0:(client.c:1683:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572599.504737:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572599.504742:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572599.504753:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572599.504757:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 1 seconds 00000100:00000001:0.0:1788572599.504762:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572599.504764:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.513745:0:266398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572600.513753:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572600.513757:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572600.513764:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 1 seconds 00000100:00000001:0.0:1788572600.513769:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572600.513774:0:266398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572600.513779:0:266398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572600.513784:0:266398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572600.513791:0:266398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572600.513796:0:266398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572600.513803:0:266398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.513806:0:266398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1788572600.513818:0:266398:0:(niobuf.c:948:ptl_send_rpc()) @@@ Allocating new XID for resend on EINPROGRESS req@ffff9c371ec71880 x1875440809988736/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572684 ref 2 fl Rpc:RSQU/200/0 rc 0/-115 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788572600.513863:0:266398:0:(client.c:3749:ptlrpc_set_mbits()) resend with new mbits old x1875440809988096 new x1875440809989120 02000000:00000001:0.0:1788572600.513868:0:266398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572600.513873:0:266398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572600.513886:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572600.513894:0:266398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788572600.513905:0:266398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572600.513913:0:266398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809988736, portal 10 00000100:00000001:0.0:1788572600.513916:0:266398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572600.513919:0:266398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286339712 : -109714423211904 : ffff9c371ec71880) 00000100:00000040:0.0:1788572600.513927:0:266398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=202 req@ffff9c371ec71880 x1875440809988736/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572685 ref 3 fl Rpc:r/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572600.513943:0:266398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572600.513949:0:266398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572600.513955:0:266398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572600.513960:0:266398:0:(niobuf.c:94:ptl_send_buf()) Sending 496 bytes to portal 12, xid 1875440809988736, offset 0 00000400:00000200:0.0:1788572600.513972:0:266398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572600.513988:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572600.514000:0:266398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572600.514008:0:266398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572600.514012:0:266398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572600.514018:0:266398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572600.514022:0:266398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572600.514031:0:266398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540072:540072:256:4294967295] 192.168.202.148@tcp LPNI seq info [540037:540037:8:4294967295] 00000400:00000200:0.0:1788572600.514038:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572600.514053:0:266398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572600.514063:0:266398:0:(socklnd_cb.c:931:ksocknal_send()) sending 496 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572600.514070:0:266398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788572600.514081:0:266398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572600.514089:0:266398:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572600.514096:0:266398:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0700 type 1, nob 592 niov 1 nkiov 1 00000100:00000001:0.0:1788572600.514111:0:266398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572600.514116:0:266398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26203 usecs (27 jiffies) 00000400:00000200:0.0:1788572600.514123:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572600.514128:0:266398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307739547 00000100:00000001:0.0:1788572600.514132:0:266398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307742431 00000100:00000001:0.0:1788572600.514138:0:266398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307739547 00000100:00000001:0.0:1788572600.514143:0:266398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307742431 00000100:00000001:0.0:1788572600.514148:0:266398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572600.514154:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572600.514159:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572600.514161:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572600.514167:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 80 seconds 00000100:00000001:0.0:1788572600.514172:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572600.514177:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.514193:0:266397:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572600.514197:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572600.514201:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572600.514211:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a3c0 going to sleep for 1 seconds 00000100:00000001:0.0:1788572600.514216:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572600.514220:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572600.514228:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440809986560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 0 ref 2 fl New:PQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572600.514242:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572600.514245:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572600.514251:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572600.514254:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572600.514261:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572600.514266:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809986560:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572600.514275:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572600.514280:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572600.514284:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572600.514288:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572600.514345:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572600.514351:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572600.514360:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572600.514363:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b58. 00000400:00000010:0.0:1788572600.514368:0:266397:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff9c372d20f000. 00000100:00000200:0.0:1788572600.514372:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1875440809986560, portal 10 00000100:00000001:0.0:1788572600.514377:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572600.514379:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000040:0.0:1788572600.514388:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b6680 x1875440809986560/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572685 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572600.514406:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572600.514411:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572600.514416:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788572600.514419:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 912 bytes to portal 12, xid 1875440809986560, offset 0 00000400:00000200:0.0:1788572600.514427:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572600.514435:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572600.514444:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572600.514448:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572600.514457:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572600.514459:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572600.514461:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572600.514465:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540076:540076:253:4294967295] 192.168.202.148@tcp LPNI seq info [540041:540041:5:4294967295] 00000400:00000200:0.0:1788572600.514472:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572600.514483:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572600.514490:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 912 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572600.514496:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788572600.514503:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572600.514511:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572600.514516:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0b00 type 1, nob 1008 niov 1 nkiov 1 00000100:00000001:0.0:1788572600.514523:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572600.514527:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15051 usecs (16 jiffies) 00000400:00000200:0.0:1788572600.514535:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572600.514538:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307739536 00000100:00000001:0.0:1788572600.514541:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307742431 00000100:00000001:0.0:1788572600.514545:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307739536 00000100:00000001:0.0:1788572600.514546:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307742431 00000100:00000001:0.0:1788572600.514550:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572600.514552:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.514554:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.519568:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572600.519576:0:266398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572600.519581:0:266398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572600.519586:0:266398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788572600.519591:0:266398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31f68 00000400:00000010:0.0:1788572600.519594:0:266398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31f68. 00000100:00000001:0.0:1788572600.519597:0:266398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572600.519603:0:266398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71880 x1875440809988736/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572685 ref 2 fl Rpc:RQ/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572600.519671:0:266398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71880 x1875440809988736/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572685 ref 2 fl Rpc:RQU/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572600.519689:0:266398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572600.519693:0:266398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572600.519699:0:266398:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572600.519704:0:266398:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572600.519708:0:266398:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572600.519713:0:266398:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572600.519719:0:266398:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100400:0.0:1788572600.519731:0:266398:0:(client.c:1659:after_reply()) @@@ resending request on EINPROGRESS req@ffff9c371ec71880 x1875440809988736/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572685 ref 2 fl Rpc:RQU/202/0 rc 0/-115 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572600.528925:0:266398:0:(client.c:1683:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572600.528931:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572600.528933:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572600.528935:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=2 : 2 : 2) 00000100:00100000:0.0:1788572600.528937:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 2 seconds 00000100:00000001:0.0:1788572600.528940:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572600.528942:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.562091:0:266397:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572602.562096:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572602.562099:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572602.562105:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfc600 going to sleep for 1 seconds 00000100:00000001:0.0:1788572602.562110:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572602.562112:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572602.562121:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440809989888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 0 ref 2 fl New:PQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572602.562145:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572602.562147:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.562152:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572602.562156:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572602.562160:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572602.562166:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809989888:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572602.562174:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572602.562178:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572602.562181:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.562192:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572602.562245:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572602.562249:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572602.562253:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572602.562257:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788572602.562263:0:266397:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff9c372d20f000. 00000100:00000200:0.0:1788572602.562267:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1875440809989888, portal 10 00000100:00000001:0.0:1788572602.562269:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572602.562270:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788572602.562277:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371ec71c00 x1875440809989888/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572687 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572602.562284:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572602.562286:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572602.562290:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788572602.562293:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 912 bytes to portal 12, xid 1875440809989888, offset 0 00000400:00000200:0.0:1788572602.562298:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572602.562303:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572602.562309:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572602.562323:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572602.562330:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572602.562334:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572602.562336:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572602.562339:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540078:540078:255:4294967295] 192.168.202.148@tcp LPNI seq info [540043:540043:7:4294967295] 00000400:00000200:0.0:1788572602.562343:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572602.562349:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572602.562354:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 912 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572602.562360:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572602.562375:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572602.562385:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572602.562391:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 1008 niov 1 nkiov 1 00000100:00000001:0.0:1788572602.562399:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572602.562417:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 15523 usecs (16 jiffies) 00000400:00000200:0.0:1788572602.562428:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572602.562435:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307741584 00000100:00000001:0.0:1788572602.562443:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307742431 00000100:00000001:0.0:1788572602.562450:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307741584 00000100:00000001:0.0:1788572602.562456:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307742431 00000100:00000001:0.0:1788572602.562464:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572602.562470:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.562476:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.562505:0:266396:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572602.562510:0:266396:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572602.562515:0:266396:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572602.562523:0:266396:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c37380b6b40 going to sleep for 1 seconds 00000100:00000001:0.0:1788572602.562528:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572602.562532:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572602.562542:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.562563:0:266396:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572602.562565:0:266396:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.562570:0:266396:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572602.562573:0:266396:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572602.562576:0:266396:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572602.562581:0:266396:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266396:1875440809989632:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572602.562588:0:266396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572602.562592:0:266396:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572602.562594:0:266396:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.562598:0:266396:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572602.562604:0:266396:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572602.562611:0:266396:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572602.562619:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572602.562625:0:266396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8000. 00000400:00000010:0.0:1788572602.562732:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572602.562745:0:266396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809989632, portal 10 00000100:00000001:0.0:1788572602.562750:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572602.562752:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287362688 : -109714422188928 : ffff9c371ed6b480) 00000100:00000040:0.0:1788572602.562779:0:266396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572687 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.562793:0:266396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572602.562799:0:266396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572602.562806:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788572602.562812:0:266396:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 12, xid 1875440809989632, offset 0 00000400:00000200:0.0:1788572602.562822:0:266396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572602.562829:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572602.562837:0:266396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572602.562847:0:266396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572602.562855:0:266396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572602.562863:0:266396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572602.562871:0:266396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572602.562880:0:266396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540081:540081:255:4294967295] 192.168.202.148@tcp LPNI seq info [540046:540046:7:4294967295] 00000400:00000200:0.0:1788572602.562891:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572602.562900:0:266396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572602.562908:0:266396:0:(socklnd_cb.c:931:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572602.562914:0:266396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788572602.562924:0:266396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572602.562934:0:266396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572602.562939:0:266396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0e00 type 1, nob 600 niov 1 nkiov 1 00000100:00000001:0.0:1788572602.562943:0:266396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572602.562946:0:266396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26292 usecs (27 jiffies) 00000400:00000200:0.0:1788572602.562954:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572602.562958:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307741596 00000100:00000001:0.0:1788572602.562964:0:266396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307742431 00000100:00000001:0.0:1788572602.562969:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307741596 00000100:00000001:0.0:1788572602.562981:0:266396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307742431 00000100:00000001:0.0:1788572602.562987:0:266396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572602.562991:0:266396:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.562994:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.569447:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572602.569456:0:266398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572602.569459:0:266398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572602.569462:0:266398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37050c5688. 00000400:00000200:0.0:1788572602.569482:0:266398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1000 00000400:00000010:0.0:1788572602.569485:0:266398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1000. 00000100:00000001:0.0:1788572602.569489:0:266398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572602.569496:0:266398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71880 x1875440809990272/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQ/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572602.569509:0:266398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71880 x1875440809990272/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQU/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.569519:0:266398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572602.569522:0:266398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572602.569525:0:266398:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572602.569528:0:266398:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572602.569530:0:266398:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572602.569532:0:266398:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.569535:0:266398:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572602.569541:0:266398:0:(client.c:1659:after_reply()) @@@ resending request on EINPROGRESS req@ffff9c371ec71880 x1875440809990272/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQU/202/0 rc 0/-115 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.569550:0:266398:0:(client.c:1683:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.569552:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572602.569555:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572602.569557:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=3 : 3 : 3) 00000100:00100000:0.0:1788572602.569559:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 3 seconds 00000100:00000001:0.0:1788572602.569562:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572602.569563:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.570479:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572602.570486:0:266396:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572602.570487:0:266396:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572602.570495:0:266396:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8000. 00000400:00000200:0.0:1788572602.570499:0:266396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572602.570501:0:266396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572602.570505:0:266396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572602.570510:0:266396:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572602.570520:0:266396:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.570527:0:266396:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572602.570528:0:266396:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572602.570530:0:266396:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572602.570532:0:266396:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572602.570534:0:266396:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572602.570535:0:266396:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572602.570537:0:266396:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572602.570542:0:266396:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=16 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572602.570549:0:266396:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=77 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572602.570564:0:266396:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1788572602.570573:0:266396:0:(client.c:1517:ptlrpc_check_status()) @@@ check status: rc = -115 req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQU/200/0 rc 0/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.570589:0:266396:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000001:0.0:1788572602.570594:0:266396:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572602.570597:0:266396:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869953 00000100:00000001:0.0:1788572602.570601:0:266396:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572602.570605:0:266396:0:(client.c:1805:after_reply()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000040:0.0:1788572602.570612:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.570633:0:266396:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551501 : -115 : 0xffffffffffffff8d) 00000100:00000001:0.0:1788572602.570660:0:266396:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572602.570662:0:266396:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572602.570667:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572687 ref 2 fl Interpret:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572602.570701:0:266396:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266396:1875440809989632:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572602.570709:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572602.570714:0:266396:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000100:00000001:0.0:1788572602.570719:0:266396:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572602.570732:0:266396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572602.570738:0:266396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572687 ref 2 fl Complete:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.570753:0:266396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572602.570771:0:266396:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c37380b6b40. 00000100:00000001:0.0:1788572602.570776:0:266396:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572602.570780:0:266396:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551501 : -115 : ffffffffffffff8d) 00000002:00000040:0.0:1788572602.570787:0:266396:0:(mdc_reint.c:34:mdc_reint()) error in handling -115 00000100:00000001:0.0:1788572602.570792:0:266396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572602.570807:0:266396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b480 x1875440809989632/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/448 e 0 to 0 dl 1788572687 ref 1 fl Complete:RQU/200/0 rc -115/-115 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572602.570815:0:266396:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572602.570817:0:266396:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572602.570818:0:266396:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572602.570822:0:266396:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572602.570823:0:266396:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572602.570825:0:266396:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572602.570827:0:266396:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572602.570828:0:266396:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c373808fa00. 02000000:00000001:0.0:1788572602.570831:0:266396:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572602.570832:0:266396:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572602.570833:0:266396:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b480. 00000100:00000001:0.0:1788572602.570836:0:266396:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572602.570838:0:266396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000002:00080000:0.0:1788572602.570840:0:266396:0:(mdc_reint.c:252:mdc_create()) lustre-MDT0001-mdc-ffff9c3711155800: resend:3 create on [0x2400013a6:0xd83:0x0]/[0x2400013a6:0xd87:0x0] 00000002:00000001:0.0:1788572602.570843:0:266396:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572602.570847:0:266396:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3704ff6400 count: 2 00010000:00000001:0.0:1788572602.570855:0:266396:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572602.570857:0:266396:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572602.570859:0:266396:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572602.570861:0:266396:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572602.570863:0:266396:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572602.570866:0:266396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572602.570869:0:266396:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572602.570881:0:266396:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572602.570883:0:266396:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572602.570884:0:266396:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572602.570886:0:266396:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572602.570889:0:266396:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572602.570890:0:266396:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572602.570892:0:266396:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.570893:0:266396:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572602.570895:0:266396:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572602.570897:0:266396:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572602.570902:0:266396:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37072bb400. 00000100:00000001:0.0:1788572602.570905:0:266396:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572602.570907:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572602.570908:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572602.570910:0:266396:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572602.570911:0:266396:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572602.570912:0:266396:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000020:00100000:0.0:1788572602.570919:0:266396:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788572602.570924:0:266396:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572602.570930:0:266396:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572602.570931:0:266396:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572602.570934:0:266396:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018acc0. 00000100:00000001:0.0:1788572602.570936:0:266396:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040027840 : -109714669523776 : ffff9c371018acc0) 00000100:00000001:0.0:1788572602.570938:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572602.570963:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788572602.570966:0:266396:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572602.570974:0:266396:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.570976:0:266396:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572602.570977:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572602.570978:0:266396:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.570980:0:266396:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572602.570981:0:266396:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=3 : 3 : 3) 00000100:00100000:0.0:1788572602.570984:0:266396:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018acc0 going to sleep for 3 seconds 00000100:00000001:0.0:1788572602.570986:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572602.570987:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788572602.570988:0:266396:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572602.570990:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572603.457748:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307742431, at 4307742464 00000100:00000010:0.0:1788572603.457761:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116269c0. 00000100:00000001:0.0:1788572604.737943:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.737948:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572604.737953:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572604.737963:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.737977:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572604.737980:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.737985:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572604.737989:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572604.737994:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572604.738000:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:874b6a47-e66d-4b10-8108-059b044e1170:2403:1875440809991296:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.738020:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572604.738027:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572604.738031:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.738036:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572604.738044:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c370137f400. 02000000:00000001:0.0:1788572604.738048:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572604.738060:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572604.738068:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8000. 00000400:00000010:0.0:1788572604.738079:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572604.738087:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809991296, portal 25 00000100:00000001:0.0:1788572604.738092:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572604.738101:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291929344 : -109714417622272 : ffff9c371f1c6300) 00000100:00000040:0.0:1788572604.738112:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738132:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572604.738137:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572604.738141:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572604.738146:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875440809991296, offset 0 00000400:00000200:0.0:1788572604.738157:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738168:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738180:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572604.738186:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572604.738192:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572604.738205:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572604.738210:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738215:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540082:540082:256:4294967295] 192.168.202.148@tcp LPNI seq info [540047:540047:8:4294967295] 00000400:00000200:0.0:1788572604.738220:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738231:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572604.738237:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572604.738242:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572604.738250:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572604.738260:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572604.738264:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572604.738281:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572604.738288:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572604.738293:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307747550 00000100:00000010:0.0:1788572604.738301:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116269c0. 00000100:00000001:0.0:1788572604.738309:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572604.738318:0:2403:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307747550 00000100:00000010:0.0:1788572604.738325:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626c60. 00000100:00000001:0.0:1788572604.738331:0:2403:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572604.738343:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.738347:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738351:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738355:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.738363:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572604.738366:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572604.738371:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.738373:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572604.738375:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572604.738383:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738395:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572604.738399:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.738405:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572604.738407:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572604.738410:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572604.738416:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809991552:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.738422:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572604.738427:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572604.738430:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.738434:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572604.738440:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c370137ec00. 02000000:00000001:0.0:1788572604.738444:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572604.738451:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572604.738456:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8e18. 00000400:00000010:0.0:1788572604.738460:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572604.738463:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809991552, portal 10 00000100:00000001:0.0:1788572604.738466:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572604.738469:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788572604.738476:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738488:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572604.738493:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572604.738501:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572604.738507:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809991552, offset 0 00000400:00000200:0.0:1788572604.738516:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738520:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738525:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572604.738530:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572604.738536:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572604.738543:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572604.738549:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738557:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540084:540084:254:4294967295] 192.168.202.148@tcp LPNI seq info [540049:540049:6:4294967295] 00000400:00000200:0.0:1788572604.738564:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738578:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572604.738585:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572604.738590:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572604.738597:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572604.738608:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572604.738613:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572604.738618:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572604.738623:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26472 usecs (27 jiffies) 00000400:00000200:0.0:1788572604.738628:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572604.738631:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307743771 00000100:00000001:0.0:1788572604.738634:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572604.738682:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307743772 00000100:00000001:0.0:1788572604.738688:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572604.738693:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.738699:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738704:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572604.738712:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738750:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572604.738754:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.738762:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572604.738769:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572604.738773:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572604.738779:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809991808:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.738783:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572604.738786:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572604.738788:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.738791:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572604.738795:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3711c21c00. 02000000:00000001:0.0:1788572604.738796:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572604.738799:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572604.738801:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8898. 00000400:00000010:0.0:1788572604.738804:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788572604.738806:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809991808, portal 4 00000100:00000001:0.0:1788572604.738807:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572604.738808:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291920384 : -109714417631232 : ffff9c371f1c4000) 00000100:00000040:0.0:1788572604.738812:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738816:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572604.738818:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572604.738819:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572604.738821:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809991808, offset 0 00000400:00000200:0.0:1788572604.738823:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738825:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738828:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572604.738829:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572604.738836:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572604.738838:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572604.738839:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572604.738845:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540086:540086:252:4294967295] 192.168.202.148@tcp LPNI seq info [540051:540051:4:4294967295] 00000400:00000200:0.0:1788572604.738848:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572604.738852:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572604.738855:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572604.738857:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572604.738860:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572604.738862:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572604.738864:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572604.738866:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572604.738868:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 169510 usecs (170 jiffies) 00000400:00000200:0.0:1788572604.738870:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572604.738871:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307743915 00000100:00000001:0.0:1788572604.738873:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572604.738874:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307743915 00000100:00000001:0.0:1788572604.738875:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572604.738876:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.738877:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738879:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738880:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.738882:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572604.738883:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572604.738884:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.738885:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572604.738886:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738888:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572604.738921:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572604.738923:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788572604.738928:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572604.738931:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572604.738935:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788572604.738936:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788572604.738943:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572604.738946:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738953:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.738955:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738959:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738960:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572604.738967:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572604.738968:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9600. 00000400:00000200:0.0:1788572604.738971:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572604.738973:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572604.738974:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f36e8 00000400:00000010:0.0:1788572604.738975:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f36e8. 00000100:00000001:0.0:1788572604.738977:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572604.738979:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4380 x1875440809991424/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738983:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.738984:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440809991424/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.738989:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.738990:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572604.738997:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572604.738999:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572604.739000:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572604.739003:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572604.739006:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31d48 00000400:00000010:0.0:1788572604.739007:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31d48. 00000100:00000001:0.0:1788572604.739008:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572604.739011:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739018:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.739027:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739032:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.739034:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572604.739043:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572604.739045:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9e00. 00000400:00000200:0.0:1788572604.739047:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572604.739050:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572604.739052:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572604.739054:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572604.739056:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572604.739059:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5500 x1875440809991680/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739065:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.739067:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440809991680/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739070:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.739071:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572604.739116:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572604.739118:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572604.739119:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572604.739121:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572604.739123:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788572604.739124:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788572604.739125:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572604.739127:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739132:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.739133:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.739137:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.739138:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572604.741848:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.741852:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572604.741865:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572604.741868:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572604.741870:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8000. 00000400:00000200:0.0:1788572604.741873:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31cc0 00000400:00000010:0.0:1788572604.741874:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31cc0. 00000100:00000001:0.0:1788572604.741876:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572604.741882:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572604.741895:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.741901:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572604.741903:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.741909:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572604.741913:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572604.741917:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572604.741919:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.741923:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572604.741931:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c373030dc60 time=19 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572604.741938:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c373030dc18 time=19 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572604.741945:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572604.741947:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572604.741952:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572604.741955:0:2403:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.741965:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572604.741976:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.741992:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.741997:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572604.742000:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572604.742008:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572604.742029:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:874b6a47-e66d-4b10-8108-059b044e1170:2403:1875440809991296:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.742047:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742053:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.742058:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440809991296/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742074:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572604.742078:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572604.742082:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c370137f400. 02000000:00000001:0.0:1788572604.742089:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572604.742093:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572604.742097:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572604.742100:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572604.742104:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9400. 02000000:00000001:0.0:1788572604.742111:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572604.742114:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572604.742117:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788572604.742123:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572604.742127:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742131:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742135:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.742139:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572604.742150:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572604.742153:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572604.742157:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8e18. 00000400:00000200:0.0:1788572604.742159:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31dd0 00000400:00000010:0.0:1788572604.742161:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31dd0. 00000100:00000001:0.0:1788572604.742163:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572604.742177:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572604.742204:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742229:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572604.742231:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742233:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572604.742234:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572604.742242:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572604.742245:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.742249:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572604.742256:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707ccf460 time=18 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572604.742264:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccf418 time=79 v=5 (1 1 1 0) 00000100:00000001:0.0:1788572604.742269:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572604.742272:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572604.742276:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572604.742280:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742285:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572604.742289:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9c3711155800: skip recheck: last_committed 17179869953 00000100:00000001:0.0:1788572604.742291:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572604.742292:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572604.742298:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742313:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.742316:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572604.742317:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572604.742321:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572604.742329:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809991552:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.742338:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742342:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.742348:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440809991552/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572689 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742365:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572604.742368:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572604.742371:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c370137ec00. 02000000:00000001:0.0:1788572604.742375:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572604.742378:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572604.742381:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572604.742385:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572604.742394:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9800. 02000000:00000001:0.0:1788572604.742398:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572604.742401:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572604.742405:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788572604.742409:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572604.742412:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742416:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742420:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.742422:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572604.742427:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572604.742429:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572604.742433:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8898. 00000400:00000200:0.0:1788572604.742438:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31110 00000400:00000010:0.0:1788572604.742441:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31110. 00000100:00000001:0.0:1788572604.742446:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572604.742452:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572604.742474:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742485:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572604.742487:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742491:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572604.742493:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572604.742495:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572604.742498:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572604.742503:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572604.742509:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c372073d460 time=18 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572604.742514:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c372073d418 time=141 v=5 (1 0 0 0) 00000100:00000001:0.0:1788572604.742516:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572604.742517:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572604.742519:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572604.742520:0:2403:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742522:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00100000:0.0:1788572604.742524:0:2403:0:(client.c:3133:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: committing for last_committed 4295010077 gen 7 00000100:00000040:0.0:1788572604.742530:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 4295010077) req@ffff9c371ec71f80 x1875440809982336/t4295010077(4295010077) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572604.742563:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.742569:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440809982336/t4295010077(4295010077) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572614 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572604.742597:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572604.742600:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572604.742604:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788572604.742608:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572604.742610:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572604.742613:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572604.742617:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572604.742619:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3731384800. 02000000:00000001:0.0:1788572604.742624:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572604.742627:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572604.742629:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788572604.742669:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572604.742681:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742684:0:2403:0:(client.c:3196:ptlrpc_free_committed()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788572604.742688:0:2403:0:(client.c:3243:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9c3711155800: normal: skip=1 replay=1/1 committed=0/0 00000100:00000001:0.0:1788572604.742691:0:2403:0:(client.c:3244:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572604.742694:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572604.742703:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742720:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572604.742724:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572604.742727:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572604.742741:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572604.742760:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809991808:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572604.742767:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742771:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572604.742783:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4000 x1875440809991808/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572620 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572604.742801:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572604.742804:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572604.742808:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3711c21c00. 02000000:00000001:0.0:1788572604.742817:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572604.742821:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572604.742824:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572604.742828:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572604.742831:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9000. 02000000:00000001:0.0:1788572604.742834:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572604.742838:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572604.742842:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4000. 00000100:00000001:0.0:1788572604.742847:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572604.742850:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742854:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572604.742862:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572604.742865:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572604.742871:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572604.742874:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.569700:0:266398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572605.569706:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.569709:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572605.569713:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 1 seconds 00000100:00000001:0.0:1788572605.569716:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.569720:0:266398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.569722:0:266398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572605.569724:0:266398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.569735:0:266398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572605.569738:0:266398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572605.569744:0:266398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.569747:0:266398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1788572605.569759:0:266398:0:(niobuf.c:948:ptl_send_rpc()) @@@ Allocating new XID for resend on EINPROGRESS req@ffff9c371ec71880 x1875440809991936/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572687 ref 2 fl Rpc:RSQU/202/0 rc 0/-115 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788572605.569768:0:266398:0:(client.c:3749:ptlrpc_set_mbits()) resend with new mbits old x1875440809990400 new x1875440809992064 02000000:00000001:0.0:1788572605.569776:0:266398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572605.569786:0:266398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572605.569793:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572605.569797:0:266398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8898. 00000400:00000010:0.0:1788572605.569804:0:266398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788572605.569810:0:266398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809991936, portal 10 00000100:00000001:0.0:1788572605.569813:0:266398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.569815:0:266398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286339712 : -109714423211904 : ffff9c371ec71880) 00000100:00000040:0.0:1788572605.569822:0:266398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=202 req@ffff9c371ec71880 x1875440809991936/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.569828:0:266398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572605.569831:0:266398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572605.569835:0:266398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788572605.569837:0:266398:0:(niobuf.c:94:ptl_send_buf()) Sending 496 bytes to portal 12, xid 1875440809991936, offset 0 00000400:00000200:0.0:1788572605.569842:0:266398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572605.569849:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.569855:0:266398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572605.569860:0:266398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572605.569864:0:266398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572605.569867:0:266398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572605.569870:0:266398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572605.569874:0:266398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540087:540087:256:4294967295] 192.168.202.148@tcp LPNI seq info [540052:540052:8:4294967295] 00000400:00000200:0.0:1788572605.569878:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.569886:0:266398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572605.569892:0:266398:0:(socklnd_cb.c:931:ksocknal_send()) sending 496 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572605.569897:0:266398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572605.569903:0:266398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572605.569910:0:266398:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572605.569914:0:266398:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 592 niov 1 nkiov 1 00000100:00000001:0.0:1788572605.569928:0:266398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.569932:0:266398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 26471 usecs (27 jiffies) 00000400:00000200:0.0:1788572605.569935:0:266398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572605.569943:0:266398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307744603 00000100:00000001:0.0:1788572605.569946:0:266398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.569949:0:266398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307744603 00000100:00000001:0.0:1788572605.569951:0:266398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.569953:0:266398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.569956:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.569958:0:266398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.569959:0:266398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572605.569962:0:266398:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c372dbfce40 going to sleep for 80 seconds 00000100:00000001:0.0:1788572605.569964:0:266398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.569965:0:266398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572605.572548:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.572560:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.572564:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.572565:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.572571:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.572577:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.572583:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.572587:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.572591:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.572594:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 440 into portal 10 MB=0x6a9b41746df80 00000400:00000200:0.0:1788572605.572598:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.572602:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 440/440 into md 0x52aba1 [1] + 224 00000800:00000001:0.0:1788572605.572609:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.572615:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.572617:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.572620:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.572625:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.572630:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ec71880 x1875440809991936/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572690 ref 2 fl Rpc:Qr/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572605.572680:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=202 mlen=440 offset=224 replen=440 req@ffff9c371ec71880 x1875440809991936/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 496/440 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RQ/202/ffffffff rc 0/-1 job:'ln.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.572705:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.572712:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.572715:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.633715:0:266396:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572605.633734:0:266396:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.633736:0:266396:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572605.633742:0:266396:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018acc0 going to sleep for 1 seconds 00000100:00000001:0.0:1788572605.633746:0:266396:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.633749:0:266396:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572605.633765:0:266396:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.633813:0:266396:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572605.633819:0:266396:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.633827:0:266396:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572605.633833:0:266396:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572605.633844:0:266396:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572605.633857:0:266396:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job mkdir:5fa795d2-07a0-4260-8bbc-bb3cde572406:266396:1875440809991168:192.168.202.148@tcp:36:mkdir.0 00000100:00000001:0.0:1788572605.633875:0:266396:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572605.633883:0:266396:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572605.633887:0:266396:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.633894:0:266396:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572605.633903:0:266396:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572605.633914:0:266396:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572605.633936:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572605.633948:0:266396:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8e18. 00000400:00000010:0.0:1788572605.633961:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788572605.633973:0:266396:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809991168, portal 10 00000100:00000001:0.0:1788572605.633984:0:266396:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.633988:0:266396:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000040:0.0:1788572605.634002:0:266396:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634026:0:266396:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572605.634045:0:266396:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572605.634054:0:266396:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572605.634065:0:266396:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 12, xid 1875440809991168, offset 0 00000400:00000200:0.0:1788572605.634082:0:266396:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572605.634103:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.634113:0:266396:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572605.634120:0:266396:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572605.634130:0:266396:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572605.634138:0:266396:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572605.634146:0:266396:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572605.634157:0:266396:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540088:540088:256:4294967295] 192.168.202.148@tcp LPNI seq info [540053:540053:8:4294967295] 00000400:00000200:0.0:1788572605.634173:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.634199:0:266396:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572605.634224:0:266396:0:(socklnd_cb.c:931:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572605.634233:0:266396:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572605.634243:0:266396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572605.634254:0:266396:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572605.634258:0:266396:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 600 niov 1 nkiov 1 00000100:00000001:0.0:1788572605.634274:0:266396:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.634282:0:266396:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27006 usecs (28 jiffies) 00000400:00000200:0.0:1788572605.634291:0:266396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572605.634299:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307744668 00000100:00000001:0.0:1788572605.634308:0:266396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.634315:0:266396:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307744668 00000100:00000001:0.0:1788572605.634320:0:266396:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.634328:0:266396:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.634335:0:266396:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.634341:0:266396:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572605.634388:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.634396:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572605.634415:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.634436:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.634448:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788572605.634454:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788572605.634466:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.634481:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634492:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.634501:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572690 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634513:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.634516:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572605.634533:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.634537:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572605.634542:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.634549:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.634554:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572605.634557:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572605.634562:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.634568:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5c00 x1875440809992320/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/202/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634579:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.634581:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b5c00 x1875440809992320/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 3 fl Rpc:Qr/202/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634587:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.634588:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572605.634676:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.634680:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5500. 00000400:00000200:0.0:1788572605.634684:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.634692:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.634698:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf13b8 00000400:00000010:0.0:1788572605.634703:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf13b8. 00000100:00000001:0.0:1788572605.634709:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.634723:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68e00 x1875440809992192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 3 fl Rpc:r/202/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634741:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.634748:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68e00 x1875440809992192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 3 fl Rpc:Qr/202/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.634763:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.634769:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572605.634879:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.634883:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9900. 00000400:00000200:0.0:1788572605.634886:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.634890:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.634893:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572605.634895:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572605.634897:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.634901:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 3 fl Rpc:Pr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.634908:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.634911:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 3 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.634917:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.634918:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572605.642095:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642115:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.642120:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642124:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.642142:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642152:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.642163:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642169:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642175:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642179:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746e180 00000400:00000200:0.0:1788572605.642185:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642191:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52abc1 [1] + 224 00000800:00000001:0.0:1788572605.642217:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642222:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.642226:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.642231:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.642239:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.642249:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5c00 x1875440809992320/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 2 fl Rpc:Qr/202/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572605.642260:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=202 mlen=456 offset=224 replen=456 req@ffff9c372d6b5c00 x1875440809992320/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RQ/202/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.642281:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.642296:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642302:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.642304:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642307:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.642311:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642315:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.642320:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642322:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642325:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642330:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 448 into portal 10 MB=0x6a9b41746dc00 00000400:00000200:0.0:1788572605.642335:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642347:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 448/448 into md 0x52abb1 [1] + 224 00000800:00000001:0.0:1788572605.642350:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642358:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.642363:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.642369:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.642386:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.642395:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572690 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572605.642436:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=448 offset=224 replen=576 req@ffff9c372d6b6d80 x1875440809991168/t0(0) o36->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 504/576 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.642509:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.642518:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642524:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.642888:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642898:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.642902:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.642909:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.642914:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642922:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.642929:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642936:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642947:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642961:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 664 into portal 10 MB=0x6a9b41746db00 00000400:00000200:0.0:1788572605.642968:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.642982:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 664/664 into md 0x52abd9 [32] + 224 00000800:00000001:0.0:1788572605.642999:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.643020:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.643029:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.643045:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.643061:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.643073:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 2 fl Rpc:PQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572605.643099:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=664 offset=224 replen=66584 req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.643132:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.643138:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.643143:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.643159:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572605.643171:0:266397:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572605.643175:0:266397:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572605.643181:0:266397:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2210. 00000400:00000200:0.0:1788572605.643189:0:266397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c373233dc00 00000400:00000010:0.0:1788572605.643202:0:266397:0:(lib-lnet.h:286:lnet_md_free()) kfreed 'md': 632 at ffff9c373233dc00. 00000100:00000001:0.0:1788572605.643212:0:266397:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.643222:0:266397:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RPQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572605.643257:0:266397:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440809990912/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/66584 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RPQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.643285:0:266397:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572605.643290:0:266397:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.643297:0:266397:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572605.643302:0:266397:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572605.643305:0:266397:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572605.643310:0:266397:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.643318:0:266397:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572605.643332:0:266397:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=19 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572605.643346:0:266397:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=51 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572605.643355:0:266397:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572605.643360:0:266397:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.643366:0:266397:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572605.643375:0:266397:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.643382:0:266397:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572605.643386:0:266397:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x60000dad9/0x1] 00000100:00000001:0.0:1788572605.643392:0:266397:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572605.643397:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.643401:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000001:0.0:1788572605.643408:0:266397:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572605.643411:0:266397:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572605.643419:0:266397:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572605.643424:0:266397:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.643433:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/664 e 0 to 0 dl 1788572690 ref 3 fl Rpc:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.643462:0:266397:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.643469:0:266397:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572605.643474:0:266397:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572605.643489:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/664 e 0 to 0 dl 1788572690 ref 3 fl Interpret:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788572605.643534:0:266397:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809990912:192.168.202.148@tcp:101:touch.0 00000100:00000001:0.0:1788572605.643545:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.643551:0:266397:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.643559:0:266397:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572605.643563:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.643573:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/664 e 0 to 0 dl 1788572690 ref 3 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.643602:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572605.643610:0:266397:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c37380b6840. 00000100:00000001:0.0:1788572605.643617:0:266397:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572605.643619:0:266397:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.643624:0:266397:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572605.643627:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.643631:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.643675:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=4 00000020:00000001:0.0:1788572605.643678:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.643681:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000040:0.0:1788572605.643687:0:266397:0:(ldlm_request.c:746:ldlm_cli_enqueue_fini()) local: ffff9c3704d18c00, remote cookie: 0xd9894672d99a3290, flags: 0x1001 00010000:00010000:0.0:1788572605.643694:0:266397:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/0,1 mode: --/CW res: [0x200002b17:0xdfa:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572605.643707:0:266397:0:(ldlm_request.c:768:ldlm_cli_enqueue_fini()) remote intent success, locking [0x200002b17:0xdff:0x0].0x0, instead of [0x200002b17:0xdfa:0x0].0x0 00010000:00000001:0.0:1788572605.643712:0:266397:0:(ldlm_lock.c:527:ldlm_lock_change_resource()) Process entered 00010000:00000010:0.0:1788572605.643722:0:266397:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46b00. 00010000:00000010:0.0:1788572605.643733:0:266397:0:(ldlm_resource.c:1487:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff9c372d676300. 00010000:00000001:0.0:1788572605.643742:0:266397:0:(ldlm_lock.c:583:ldlm_lock_change_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572605.643750:0:266397:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/0,1 mode: --/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572605.643790:0:266397:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/0,1 mode: --/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.643811:0:266397:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1788572605.643819:0:266397:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1788572605.643824:0:266397:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1788572605.643828:0:266397:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1788572605.643835:0:266397:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1788572605.643840:0:266397:0:(ldlm_resource.c:1876:ldlm_resource_dump()) --- Resource: [0x200002b17:0xdff:0x0].0x0 (ffff9c3713c46b00) refcount = 1 00010000:00010000:0.0:1788572605.643846:0:266397:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/0,1 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.643872:0:266397:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00010000:00000001:0.0:1788572605.643882:0:266397:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1788572605.643885:0:266397:0:(ldlm_lock.c:1914:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572605.643894:0:266397:0:(ldlm_request.c:286:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1788572605.643897:0:266397:0:(ldlm_request.c:295:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572605.643902:0:266397:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/0,1 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.643911:0:266397:0:(ldlm_request.c:842:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1788572605.643912:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.643913:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.643914:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.643915:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.643917:0:266397:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.643923:0:266397:0:(mdc_locks.c:780:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1788572605.643927:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.643928:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.643933:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.643935:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.643937:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.643949:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.643951:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.643957:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=4 00000020:00000001:0.0:1788572605.643962:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.643966:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.643970:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572605.643977:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572605.643985:0:266397:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 5/1,1 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.644004:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644008:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.644013:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644017:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644023:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=5 00000020:00000001:0.0:1788572605.644029:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644036:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644043:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572605.644049:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 5/1,1 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.644064:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644067:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572605.644071:0:266397:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 4/1,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.644093:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572605.644096:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644098:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.644101:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644104:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1788572605.644110:0:266397:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/664 e 0 to 0 dl 1788572690 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572605.644143:0:266397:0:(mdc_request.c:760:mdc_set_open_replay_data()) Process entered 00000002:00100000:0.0:1788572605.644151:0:266397:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 912/664 e 0 to 0 dl 1788572690 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572605.644157:0:266397:0:(mdc_request.c:812:mdc_set_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.644170:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644172:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644176:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.644182:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644190:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00000002:00010000:0.0:1788572605.644198:0:266397:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00000010:0.0:1788572605.644216:0:266397:0:(mdc_locks.c:941:mdc_finish_enqueue()) kmalloced '(lmm)': 56 at ffff9c372e9a9f80. 00010000:00000001:0.0:1788572605.644223:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644227:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572605.644232:0:266397:0:(mdc_locks.c:983:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.644237:0:266397:0:(mdc_locks.c:1188:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.644243:0:266397:0:(mdc_locks.c:1290:mdc_finish_intent_lock()) Process entered 00000100:00000001:0.0:1788572605.644246:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.644248:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000001:0.0:1788572605.644252:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.644253:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00010000:00000001:0.0:1788572605.644255:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644258:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644262:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.644264:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644267:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00000002:00010000:0.0:1788572605.644273:0:266397:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.644290:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644292:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.644296:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1788572605.644297:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644299:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644304:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.644307:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644312:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000040:0.0:1788572605.644321:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 2 00010000:00000001:0.0:1788572605.644327:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1788572605.644332:0:266397:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9c371ea5a000 type 13 mode 4 res 8589945623/3583 (0 0) 00010000:00000001:0.0:1788572605.644334:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644335:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572605.644336:0:266397:0:(mdc_locks.c:1385:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1788572605.644338:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf768 00000002:00002000:0.0:1788572605.644339:0:266397:0:(mdc_locks.c:1390:mdc_finish_intent_lock()) D_IT dentry=f13 intent=open|creat status=0 disp=0x1800037: rc = 0 00000002:00000001:0.0:1788572605.644341:0:266397:0:(mdc_locks.c:1545:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572605.644343:0:266397:0:(lmv_intent.c:429:lmv_intent_open()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572605.644345:0:266397:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1788572605.644348:0:266397:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b17:0xdfa:0x0] suppgids 0 -1: rc 0 00000080:00000001:0.0:1788572605.644350:0:266397:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.644353:0:266397:0:(namei.c:668:ll_lookup_it_finish()) Process entered 00000080:00002000:0.0:1788572605.644354:0:266397:0:(namei.c:674:ll_lookup_it_finish()) it ffff9c3711626180 it_disposition 1800037 00000080:00002000:0.0:1788572605.644360:0:266397:0:(namei.c:653:ll_splice_alias()) Add dentry ffff9c37034e2948 inode 0000000000000000 refc 1 flags 0xe 00000080:00000001:0.0:1788572605.644364:0:266397:0:(namei.c:830:ll_lookup_it_finish()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572605.644367:0:266397:0:(namei.c:1624:ll_create_it()) Process entered 00000020:00000001:0.0:1788572605.644369:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf990 00000080:00200000:0.0:1788572605.644370:0:266397:0:(namei.c:1628:ll_create_it()) VFS Op:name=f13, dir=[0x200002b17:0xdfa:0x0](ffff9c37081ff448), intent=open|creat 00000080:00000001:0.0:1788572605.644374:0:266397:0:(namei.c:1575:ll_create_node()) Process entered 00000080:00000001:0.0:1788572605.644376:0:266397:0:(llite_lib.c:3840:ll_prep_inode()) Process entered 00000002:00000001:0.0:1788572605.644379:0:266397:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00000002:00000001:0.0:1788572605.644383:0:266397:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1788572605.644385:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329151 : 144115373145329151 : 200002b17000dff) 00000080:00000001:0.0:1788572605.644392:0:266397:0:(namei.c:113:ll_iget()) Process entered 00000080:00000010:0.0:1788572605.644492:0:266397:0:(super25.c:42:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff9c3708706300. 00000020:00000001:0.0:1788572605.644504:0:266397:0:(class_obd.c:978:obd_heat_clear()) Process entered 00000020:00000001:0.0:1788572605.644506:0:266397:0:(class_obd.c:981:obd_heat_clear()) Process leaving 00000080:00000001:0.0:1788572605.644509:0:266397:0:(llite_lib.c:3353:ll_read_inode2()) Process entered 00000080:00200000:0.0:1788572605.644511:0:266397:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200002b17:0xdff:0x0](ffff9c37087063c8) 00000020:00000001:0.0:1788572605.644514:0:266397:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572605.644517:0:266397:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572605.644519:0:266397:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572605.644523:0:266397:0:(lu_object.c:830:lu_object_find_at()) Process entered 00000080:00000010:0.0:1788572605.644527:0:266397:0:(vvp_object.c:304:vvp_object_alloc()) slab-alloced 'vob': 200 at ffff9c37322140c8. 00000020:00000001:0.0:1788572605.644530:0:266397:0:(cl_object.c:60:cl_object_header_init()) Process entered 00000020:00000001:0.0:1788572605.644531:0:266397:0:(cl_object.c:68:cl_object_header_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.644537:0:266397:0:(lov_object.c:2359:lov_object_alloc()) Process entered 00020000:00000010:0.0:1788572605.644541:0:266397:0:(lov_object.c:2360:lov_object_alloc()) slab-alloced 'lov': 248 at ffff9c3730ca38b8. 00020000:00000001:0.0:1788572605.644543:0:266397:0:(lov_object.c:2374:lov_object_alloc()) Process leaving (rc=18446634359588534456 : -109714121017160 : ffff9c3730ca38b8) 00020000:00000001:0.0:1788572605.644546:0:266397:0:(lov_object.c:1410:lov_object_init()) Process entered 00020000:00000001:0.0:1788572605.644548:0:266397:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572605.644569:0:266397:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626d80. 00020000:00000010:0.0:1788572605.644573:0:266397:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c372ecca780. 00020000:00000010:0.0:1788572605.644578:0:266397:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626300. 00020000:00000001:0.0:1788572605.644580:0:266397:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061635840 : -109714647915776 : ffff9c3711626300) 00020000:00000002:0.0:1788572605.644583:0:266397:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626300, objid 0xdff:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572605.644587:0:266397:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572605.644591:0:266397:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c372ecca780: ostid: 0x2c0000401:21020 ost idx: 1 gen: 0 00020000:00000001:0.0:1788572605.644593:0:266397:0:(lov_object.c:598:lov_init_composite()) Process entered 00020000:00000002:0.0:1788572605.644595:0:266397:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626300, objid 0xdff:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 2, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572605.644599:0:266397:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572605.644602:0:266397:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c372ecca780: ostid: 0x2c0000401:21020 ost idx: 1 gen: 0 00020000:00000010:0.0:1788572605.644606:0:266397:0:(lov_object.c:618:lov_init_composite()) kmalloced '(comp->lo_mirrors)': 12 at ffff9c3731c907a0. 00020000:00000010:0.0:1788572605.644609:0:266397:0:(lov_object.c:622:lov_init_composite()) kmalloced '(comp->lo_entries)': 184 at ffff9c371018af00. 00020000:00000001:0.0:1788572605.644613:0:266397:0:(lov_object.c:189:lov_init_raid0()) Process entered 00020000:00000010:0.0:1788572605.644623:0:266397:0:(lov_object.c:194:lov_init_raid0()) kmalloced '(r0->lo_sub)': 8 at ffff9c371eb746a0. 00020000:00000001:0.0:1788572605.644626:0:266397:0:(lov_object.c:83:lov_sub_find()) Process entered 00000020:00000001:0.0:1788572605.644628:0:266397:0:(lu_object.c:830:lu_object_find_at()) Process entered 00020000:00000001:0.0:1788572605.644633:0:266397:0:(lovsub_object.c:176:lovsub_object_alloc()) Process entered 00020000:00000010:0.0:1788572605.644678:0:266397:0:(lovsub_object.c:177:lovsub_object_alloc()) slab-alloced 'los': 200 at ffff9c37322147d0. 00000020:00000001:0.0:1788572605.644681:0:266397:0:(cl_object.c:60:cl_object_header_init()) Process entered 00000020:00000001:0.0:1788572605.644683:0:266397:0:(cl_object.c:68:cl_object_header_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.644685:0:266397:0:(lovsub_object.c:190:lovsub_object_alloc()) Process leaving (rc=18446634359611017296 : -109714098534320 : ffff9c3732214850) 00020000:00000001:0.0:1788572605.644690:0:266397:0:(lovsub_object.c:32:lovsub_object_init()) Process entered 00000008:00000010:0.0:1788572605.644694:0:266397:0:(osc_object.c:435:osc_object_alloc()) slab-alloced 'osc': 448 at ffff9c3713d8b500. 00020000:00000001:0.0:1788572605.644696:0:266397:0:(lovsub_object.c:41:lovsub_object_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.644701:0:266397:0:(lu_object.c:929:lu_object_find_at()) Process leaving (rc=18446634359611017296 : -109714098534320 : ffff9c3732214850) 00020000:00000001:0.0:1788572605.644704:0:266397:0:(lov_object.c:87:lov_sub_find()) Process leaving (rc=18446634359611017296 : -109714098534320 : ffff9c3732214850) 00020000:00000002:0.0:1788572605.644708:0:266397:0:(lov_object.c:134:lov_init_sub()) [0x2c0000401:0x521c:0x0]@ffff9c37322147d0[0:0] -> [0x200002b17:0xdff:0x0]@ffff9c37322140c8: ostid: 0x2c0000401:21020 ost idx: 1 gen: 0 00020000:00000001:0.0:1788572605.644714:0:266397:0:(lov_object.c:259:lov_init_raid0()) Process leaving (rc=128 : 128 : 80) 00020000:00000001:0.0:1788572605.644717:0:266397:0:(lov_object.c:815:lov_init_composite()) Process leaving 00020000:00000001:0.0:1788572605.644719:0:266397:0:(lov_object.c:1438:lov_object_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.644732:0:266397:0:(lu_object.c:929:lu_object_find_at()) Process leaving (rc=18446634359611015496 : -109714098536120 : ffff9c3732214148) 00000020:00001000:0.0:1788572605.644735:0:266397:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572605.644740:0:266397:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115373145329151 : 144115373145329151 : 200002b17000dff) 00000080:00000001:0.0:1788572605.644743:0:266397:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554475 : 33554475 : 200002b) 00000080:00000002:0.0:1788572605.644745:0:266397:0:(llite_lib.c:3050:ll_update_inode()) setting ino 144115373145329151 mtime from 0 to 1788572599 00000080:00000001:0.0:1788572605.644750:0:266397:0:(llite_lib.c:3378:ll_read_inode2()) Process leaving 00000080:00000001:0.0:1788572605.644752:0:266397:0:(namei.c:156:ll_iget()) Process leaving (rc=18446634358911558600 : -109714797993016 : ffff9c37087063c8) 00010000:00000001:0.0:1788572605.644756:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644758:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644761:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.644763:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644766:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00000080:00000001:0.0:1788572605.644769:0:266397:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572605.644771:0:266397:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572605.644773:0:266397:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572605.644775:0:266397:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572605.644782:0:266397:0:(cl_object.c:343:cl_conf_set()) Process entered 00020000:00000001:0.0:1788572605.644785:0:266397:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572605.644786:0:266397:0:(lov_pack.c:345:lov_unpackmd()) Process entered 00020000:00000010:0.0:1788572605.644789:0:266397:0:(lov_ea.c:230:lsme_unpack()) kmalloced '(lsme)': 88 at ffff9c3711626ba0. 00020000:00000010:0.0:1788572605.644792:0:266397:0:(lov_ea.c:267:lsme_unpack()) slab-alloced 'loi': 112 at ffff9c372eccaa00. 00020000:00000010:0.0:1788572605.644801:0:266397:0:(lov_ea.c:394:lsm_unpackmd_v1v3()) kmalloced '(lsm)': 88 at ffff9c3711626540. 00020000:00000001:0.0:1788572605.644803:0:266397:0:(lov_pack.c:357:lov_unpackmd()) Process leaving (rc=18446634359061636416 : -109714647915200 : ffff9c3711626540) 00020000:00000002:0.0:1788572605.644805:0:266397:0:(lov_ea.c:755:dump_lsm()) lsm ffff9c3711626540, objid 0xdff:11031, maxbytes 0xffffffff000, magic 0x0BD10BD0, refc: 1, entry: 1, mirror: 0, flags: 0,layout_gen 0 00020000:00000002:0.0:1788572605.644809:0:266397:0:(lov_ea.c:786:dump_lsm()) [0x0, 0xffffffffffffffff): id: 0, flags: 10, magic 0x0BD10BD0, layout_gen 0, stripe count 1 (0/0), stripe size 4194304, pool: [] 00020000:00000002:0.0:1788572605.644811:0:266397:0:(lov_ea.c:798:dump_lsm()) oinfo:ffff9c372eccaa00: ostid: 0x2c0000401:21020 ost idx: 1 gen: 0 00020000:00000002:0.0:1788572605.644814:0:266397:0:(lov_object.c:1302:lov_conf_lock()) Took exclusive lov(ffff9c3730ca38b8) owner ffff9c3709594f80 00020000:00000001:0.0:1788572605.644816:0:266397:0:(lov_object.c:1508:lov_conf_set()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572605.644818:0:266397:0:(lov_object.c:1308:lov_conf_unlock()) To release exclusive lov(ffff9c3730ca38b8) owner ffff9c3709594f80 00020000:00000010:0.0:1788572605.644820:0:266397:0:(lov_ea.c:146:lsme_free()) slab-freed '(lsme->lsme_oinfo[i])': 112 at ffff9c372eccaa00. 00020000:00000010:0.0:1788572605.644823:0:266397:0:(lov_ea.c:149:lsme_free()) kvfree 'lsme': 88 at ffff9c3711626ba0. 00020000:00000010:0.0:1788572605.644825:0:266397:0:(lov_ea.c:170:lsm_free()) kfreed 'lsm': 88 at ffff9c3711626540. 00020000:00000002:0.0:1788572605.644827:0:266397:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdff:0x0] lo_layout_invalid=0 00020000:00000001:0.0:1788572605.644829:0:266397:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.644831:0:266397:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.644834:0:266397:0:(cl_object.c:448:cl_object_layout_get()) Process entered 00020000:00000002:0.0:1788572605.644836:0:266397:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000002:0.0:1788572605.644839:0:266397:0:(lov_object.c:2387:lov_lsm_addref()) lsm ffff9c3711626300 addref 2/0 by ffff9c3709594f80. 00020000:00000002:0.0:1788572605.644841:0:266397:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000001:0.0:1788572605.644843:0:266397:0:(lov_object.c:2283:lov_object_layout_get()) Process entered 00020000:00000001:0.0:1788572605.644845:0:266397:0:(lov_pack.c:242:lov_lsm_pack()) Process entered 00020000:00000001:0.0:1788572605.644846:0:266397:0:(lov_pack.c:99:lov_lsm_pack_v1v3()) Process entered 00020000:00000001:0.0:1788572605.644847:0:266397:0:(lov_pack.c:104:lov_lsm_pack_v1v3()) Process leaving (rc=56 : 56 : 38) 00020000:00000001:0.0:1788572605.644849:0:266397:0:(lov_object.c:2302:lov_object_layout_get()) Process leaving (rc=56 : 56 : 38) 00000080:00200000:0.0:1788572605.644851:0:266397:0:(file.c:6843:ll_layout_conf()) [0x200002b17:0xdff:0x0]: layout version change: 4294967294 -> 0 00000020:00001000:0.0:1788572605.644853:0:266397:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572605.644855:0:266397:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.644857:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644859:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572605.644860:0:266397:0:(llite_lib.c:3938:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1788572605.644868:0:266397:0:(lmv_obd.c:4167:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1788572605.644869:0:266397:0:(lmv_obd.c:4174:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572605.644872:0:266397:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b17:0xdff:0x0](ffff9c37087063c8) 00000080:00010000:0.0:1788572605.644875:0:266397:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200002b17:0xdff:0x0](ffff9c37087063c8) for lock 0x14415860931689c4 00800000:00000001:0.0:1788572605.644878:0:266397:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1788572605.644879:0:266397:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1788572605.644881:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.644882:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.644883:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.644885:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644887:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.644889:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.644890:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1788572605.644891:0:266397:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572605.644892:0:266397:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.644894:0:266397:0:(namei.c:1596:ll_create_node()) Process leaving 00000100:00000001:0.0:1788572605.644896:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.644903:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 3 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 4 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.644910:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000002:0.0:1788572605.644913:0:266397:0:(namei.c:1651:ll_create_it()) inode ffff9c37087063c8 need_sync_to_mds [0x200002b17:0xdff:0x0] 00000080:00000001:0.0:1788572605.644916:0:266397:0:(namei.c:1686:ll_create_it()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1788572605.644919:0:266397:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b17:0xdff:0x0](ffff9c37087063c8) 00000080:00000001:0.0:1788572605.644921:0:266397:0:(namei.c:1310:ll_lookup_it()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572605.644924:0:266397:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c3711c20600. 00000100:00000001:0.0:1788572605.644929:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.644932:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 3 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.644938:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.644941:0:266397:0:(pcc.c:3603:pcc_inode_create_fini()) Process entered 00000080:00000001:0.0:1788572605.644942:0:266397:0:(pcc.c:3606:pcc_inode_create_fini()) Process leaving (rc=0 : 0 : 0) 00000080:00000002:0.0:1788572605.644944:0:266397:0:(namei.c:1500:ll_atomic_open()) inode ffff9c37087063c8 need_sync_to_mds [0x200002b17:0xdff:0x0] 00000080:00000001:0.0:1788572605.644949:0:266397:0:(file.c:1086:ll_file_open()) Process entered 00000020:00000001:0.0:1788572605.644955:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafa68 00000080:00200008:0.0:1788572605.644957:0:266397:0:(file.c:1090:ll_file_open()) START file f13:[0x200002b17:0xdff:0x0](ffff9c37087063c8), flags 104501 00000080:00000010:0.0:1788572605.644962:0:266397:0:(file.c:70:ll_file_data_get()) slab-alloced 'lfd': 288 at ffff9c3711643e00. 00000080:00000010:0.0:1788572605.644968:0:266397:0:(file.c:1254:ll_file_open()) kmalloced '(*och_p)': 56 at ffff9c372e9a9ac0. 00000080:00000001:0.0:1788572605.644970:0:266397:0:(file.c:1008:ll_local_open()) Process entered 00800000:00000001:0.0:1788572605.644975:0:266397:0:(lmv_obd.c:4185:lmv_set_open_replay_data()) Process entered 00800000:00000001:0.0:1788572605.644977:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572605.644979:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572605.644981:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572605.644984:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572605.644985:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.644991:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdff:0x0] 00800000:00000001:0.0:1788572605.644994:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.644997:0:266397:0:(mdc_request.c:760:mdc_set_open_replay_data()) Process entered 00000002:00000010:0.0:1788572605.645002:0:266397:0:(obd.h:1465:obd_mod_alloc()) kmalloced '(mod)': 32 at ffff9c3703a9b960. 00000002:00100000:0.0:1788572605.645007:0:266397:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00000001:0.0:1788572605.645012:0:266397:0:(mdc_request.c:812:mdc_set_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572605.645014:0:266397:0:(lmv_obd.c:4191:lmv_set_open_replay_data()) Process leaving (rc=0 : 0 : 0) 00000080:00400000:0.0:1788572605.645017:0:266397:0:(rw.c:948:ras_reset()) lre 0 cr 0 cb 0 wsi 0 wp 0 nra 1 rpc 16384 r 0 csr 0 so 0 sb 0 sl 0 lr 0 00000080:00000001:0.0:1788572605.645021:0:266397:0:(file.c:1027:ll_local_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645023:0:266397:0:(file.c:1298:ll_file_open()) Process leaving via out_och_free (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.645025:0:266397:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.645028:0:266397:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 02000000:00000001:0.0:1788572605.645033:0:266397:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572605.645036:0:266397:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff9c372ec20000. 02000000:00000001:0.0:1788572605.645052:0:266397:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788572605.645054:0:266397:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.645056:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafa68 00000080:00000008:0.0:1788572605.645058:0:266397:0:(file.c:1329:ll_file_open()) COMPLETED file f13:[0x200002b17:0xdff:0x0](ffff9c37087063c8), flags 104501, rc = 0 00000080:00000001:0.0:1788572605.645062:0:266397:0:(dcache.c:174:ll_intent_release()) Process entered 00000080:00000040:0.0:1788572605.645064:0:266397:0:(dcache.c:176:ll_intent_release()) intent ffff9c3711626180 released 00000080:00010000:0.0:1788572605.645066:0:266397:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x14415860931689c4 from it ffff9c3711626180 00010000:00000001:0.0:1788572605.645073:0:266397:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572605.645074:0:266397:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572605.645076:0:266397:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c3704d18c00 refcount=3 00000020:00000001:0.0:1788572605.645078:0:266397:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.645080:0:266397:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634358850817024 : -109714858734592 : ffff9c3704d18c00) 00010000:00000001:0.0:1788572605.645082:0:266397:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572605.645086:0:266397:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 3/1,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572605.645093:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.645094:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.645096:0:266397:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1788572605.645098:0:266397:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1788572605.645101:0:266397:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1788572605.645110:0:266397:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788572605.645113:0:266397:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788572605.645114:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572605.645117:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.645118:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.645119:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645121:0:266397:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788572605.645122:0:266397:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645124:0:266397:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645126:0:266397:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645127:0:266397:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572605.645128:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.645128:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1788572605.645129:0:266397:0:(dcache.c:187:ll_intent_release()) Process leaving 00000080:00000001:0.0:1788572605.645131:0:266397:0:(pcc.c:2402:pcc_file_open()) Process entered 00000080:00000001:0.0:1788572605.645132:0:266397:0:(pcc.c:2108:pcc_may_auto_attach()) Process entered 00000080:00000001:0.0:1788572605.645133:0:266397:0:(pcc.c:2127:pcc_may_auto_attach()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572605.645135:0:266397:0:(pcc.c:2032:pcc_try_auto_attach()) Process entered 00000080:00000001:0.0:1788572605.645136:0:266397:0:(pcc.c:2038:pcc_try_auto_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645139:0:266397:0:(pcc.c:1967:pcc_try_readonly_open_attach()) Process entered 00000080:00000001:0.0:1788572605.645145:0:266397:0:(pcc.c:1970:pcc_try_readonly_open_attach()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645147:0:266397:0:(pcc.c:2435:pcc_file_open()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1788572605.645149:0:266397:0:(pcc.c:2462:pcc_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645150:0:266397:0:(namei.c:1550:ll_atomic_open()) Process leaving via out_free (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1788572605.645153:0:266397:0:(namei.c:1562:ll_atomic_open()) kfreed 'it': 72 at ffff9c3711626180. 00000080:00000001:0.0:1788572605.645155:0:266397:0:(namei.c:1564:ll_atomic_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645159:0:266397:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00200000:0.0:1788572605.645161:0:266397:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200002b17:0xdff:0x0](ffff9c37087063c8), inode mode 81a4 mask 40 00000080:00000001:0.0:1788572605.645164:0:266397:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.645186:0:266397:0:(lov_object.c:2398:lov_read_and_clear_async_rc()) Process entered 00020000:00000002:0.0:1788572605.645189:0:266397:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000002:0.0:1788572605.645192:0:266397:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000001:0.0:1788572605.645194:0:266397:0:(lov_object.c:2449:lov_read_and_clear_async_rc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645203:0:266397:0:(llite_lib.c:2418:ll_setattr_raw()) Process entered 00000020:00000001:0.0:1788572605.645205:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafc30 00000080:00200008:0.0:1788572605.645207:0:266397:0:(llite_lib.c:2427:ll_setattr_raw()) START file f13:[0x200002b17:0xdff:0x0](ffff9c37087063c8) current size 0, valid attrs 20070, mode 0, uid -1055292862, gid -1, new size 213440, atime 1788572605.000000000, mtime 1788572605.000000000, ctime 1788572605.000000000 00000080:00000002:0.0:1788572605.645212:0:266397:0:(llite_lib.c:2479:ll_setattr_raw()) setting mtime 1788572605, ctime 1788572605, now = 1788572605 00000080:00000002:0.0:1788572605.645215:0:266397:0:(llite_lib.c:2486:ll_setattr_raw()) inode ffff9c37087063c8 attr 0x100201f0 need_sync_to_mds [0x200002b17:0xdff:0x0] 00000080:00000010:0.0:1788572605.645219:0:266397:0:(llite_lib.c:2494:ll_setattr_raw()) kmalloced '(op_data)': 392 at ffff9c370395b000. 00000080:00000001:0.0:1788572605.645222:0:266397:0:(llite_lib.c:2125:ll_md_setattr()) Process entered 00800000:00000001:0.0:1788572605.645227:0:266397:0:(lmv_obd.c:3034:lmv_setattr()) Process entered 00800000:00000002:0.0:1788572605.645228:0:266397:0:(lmv_obd.c:3038:lmv_setattr()) SETATTR for [0x200002b17:0xdff:0x0], valid 0x100201f0/0x0 00800000:00000001:0.0:1788572605.645229:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572605.645230:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572605.645231:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572605.645232:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572605.645233:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.645234:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdff:0x0] 00800000:00000001:0.0:1788572605.645236:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.645238:0:266397:0:(mdc_reint.c:94:mdc_setattr()) Process entered 00000002:00000001:0.0:1788572605.645239:0:266397:0:(mdc_reint.c:54:mdc_resource_cancel_unused_res()) Process entered 00010000:00000040:0.0:1788572605.645242:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 2 00010000:00000001:0.0:1788572605.645244:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572605.645246:0:266397:0:(ldlm_request.c:2365:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1788572605.645252:0:266397:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572605.645254:0:266397:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788572605.645259:0:266397:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788572605.645261:0:266397:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788572605.645262:0:266397:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788572605.645264:0:266397:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0xdff:0x0] mode (null) 00000002:00000001:0.0:1788572605.645266:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572605.645268:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572605.645269:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572605.645270:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572605.645272:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.645273:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572605.645274:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572605.645275:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572605.645276:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572605.645277:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.645278:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572605.645279:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572605.645279:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572605.645280:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572605.645281:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.645282:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572605.645283:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572605.645283:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3713c46b00 count: 3 00010000:00000001:0.0:1788572605.645284:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572605.645285:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645286:0:266397:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645287:0:266397:0:(file.c:6808:ll_layout_conf()) Process entered 00000020:00000001:0.0:1788572605.645288:0:266397:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572605.645289:0:266397:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065795256 : -109714643756360 : ffff9c3711a1dab8) 00000020:00001000:0.0:1788572605.645290:0:266397:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1dab0 00000020:00000001:0.0:1788572605.645291:0:266397:0:(cl_object.c:343:cl_conf_set()) Process entered 00000080:00200000:0.0:1788572605.645292:0:266397:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b17:0xdff:0x0]: losing layout lock 00000080:00000001:0.0:1788572605.645294:0:266397:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered 00000080:00000001:0.0:1788572605.645295:0:266397:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving 00020000:00000001:0.0:1788572605.645299:0:266397:0:(lov_object.c:1448:lov_conf_set()) Process entered 00020000:00000001:0.0:1788572605.645300:0:266397:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1788572605.645301:0:266397:0:(lov_object.c:1535:lov_conf_set()) [0x200002b17:0xdff:0x0] lo_layout_invalid=1 00020000:00000001:0.0:1788572605.645303:0:266397:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.645304:0:266397:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572605.645305:0:266397:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1dab0 00000080:00000001:0.0:1788572605.645306:0:266397:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.645308:0:266397:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788572605.645309:0:266397:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b17:0xdff:0x0](ffff9c37087063c8) invalid 00000020:00000001:0.0:1788572605.645310:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eaf838 00000080:00002000:0.0:1788572605.645311:0:266397:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry f13 (ffff9c37034e2948) parent ffff9c37034e21b0 inode ffff9c37087063c8 refc 1 00000080:00000001:0.0:1788572605.645314:0:266397:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788572605.645315:0:266397:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788572605.645316:0:266397:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645318:0:266397:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572605.645320:0:266397:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572605.645321:0:266397:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572605.645323:0:266397:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572605.645324:0:266397:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1788572605.645326:0:266397:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c3704d18c00 with handle 0x14415860931689c4 from hash 00010000:00000001:0.0:1788572605.645328:0:266397:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572605.645329:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.645329:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.645330:0:266397:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572605.645332:0:266397:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572605.645333:0:266397:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572605.645335:0:266397:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572605.645336:0:266397:0:(ldlm_request.c:2415:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1788572605.645337:0:266397:0:(mdc_reint.c:73:mdc_resource_cancel_unused_res()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788572605.645341:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572605.645343:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 00010000:00000001:0.0:1788572605.645346:0:266397:0:(ldlm_request.c:923:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1788572605.645349:0:266397:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788572605.645356:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572605.645356:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.645357:0:266397:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.645363:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572605.645365:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572605.645368:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572605.645372:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3713f8a800. 00000100:00000001:0.0:1788572605.645377:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645378:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572605.645380:0:266397:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572605.645384:0:266397:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572605.645389:0:266397:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1788572605.645390:0:266397:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788572605.645391:0:266397:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572605.645393:0:266397:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9c3711155800 lock: ffff9c3704d18c00/0x14415860931689c4 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0xdff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xd9894672d99a3290 expref: -99 pid: 266397 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572605.645397:0:266397:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372e9a9f80. 00010000:00000040:0.0:1788572605.645399:0:266397:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46b00 count: 0 00010000:00000010:0.0:1788572605.645402:0:266397:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff9c372d676300. 00010000:00000001:0.0:1788572605.645408:0:266397:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572605.645409:0:266397:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645410:0:266397:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572605.645411:0:266397:0:(ldlm_request.c:2459:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645412:0:266397:0:(ldlm_request.c:979:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000002:0.0:1788572605.645413:0:266397:0:(mdc_reint.c:124:mdc_setattr()) setting mtime 1788572605, ctime 1788572605 00000020:00100000:0.0:1788572605.645418:0:266397:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788572605.645423:0:266397:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572605.645430:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572605.645432:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572605.645437:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a000. 00000100:00000001:0.0:1788572605.645439:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040024576 : -109714669527040 : ffff9c371018a000) 00000100:00000001:0.0:1788572605.645441:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.645448:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788572605.645452:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572605.645456:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.645458:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572605.645460:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572605.645467:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645474:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572605.645476:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.645479:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572605.645482:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572605.645484:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572605.645488:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809992832:192.168.202.148@tcp:36:touch.0 00000100:00000001:0.0:1788572605.645491:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572605.645496:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572605.645499:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.645502:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572605.645506:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572605.645508:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572605.645515:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572605.645519:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8000. 00000400:00000010:0.0:1788572605.645524:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788572605.645530:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809992832, portal 10 00000100:00000001:0.0:1788572605.645532:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.645533:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000040:0.0:1788572605.645539:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645544:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572605.645546:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572605.645548:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572605.645550:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 488 bytes to portal 12, xid 1875440809992832, offset 0 00000400:00000200:0.0:1788572605.645553:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572605.645559:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.645568:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572605.645572:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572605.645575:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572605.645578:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572605.645580:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572605.645583:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540092:540092:256:4294967295] 192.168.202.148@tcp LPNI seq info [540057:540057:8:4294967295] 00000400:00000200:0.0:1788572605.645585:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.645592:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572605.645598:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 488 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572605.645603:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572605.645610:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572605.645617:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572605.645620:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 584 niov 1 nkiov 1 00000100:00000001:0.0:1788572605.645670:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.645675:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 20695 usecs (21 jiffies) 00000400:00000200:0.0:1788572605.645680:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572605.645683:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307744673 00000100:00000001:0.0:1788572605.645686:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.645689:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307744673 00000100:00000001:0.0:1788572605.645694:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.645697:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.645700:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.645704:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.645707:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=80 : 80 : 50) 00000100:00100000:0.0:1788572605.645712:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a000 going to sleep for 80 seconds 00000100:00000001:0.0:1788572605.645717:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.645720:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572605.645785:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.645789:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572605.645794:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.645820:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.645824:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31d48 00000400:00000010:0.0:1788572605.645827:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31d48. 00000100:00000001:0.0:1788572605.645831:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.645837:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645847:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.645855:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572690 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645862:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.645863:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572605.645892:0:266399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572605.645897:0:266399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572605.645899:0:266399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572605.645901:0:266399:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37580. 00000400:00000200:0.0:1788572605.645904:0:266399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1660 00000400:00000010:0.0:1788572605.645905:0:266399:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1660. 00000100:00000001:0.0:1788572605.645908:0:266399:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.645911:0:266399:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440809992192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 2 fl Rpc:RQ/202/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572605.645917:0:266399:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440809992192/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 2 fl Rpc:RQU/202/ffffffff rc 0/-1 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645924:0:266399:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572605.645927:0:266399:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.645929:0:266399:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572605.645931:0:266399:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572605.645932:0:266399:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572605.645934:0:266399:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.645935:0:266399:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572605.645939:0:266399:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707cce5c8 time=295 v=30 (30 0 30 0) 00000100:00001000:0.0:1788572605.645942:0:266399:0:(import.c:2261:obd_at_measure()) AT ffff9c3707cce5c8 change: old=30 new=5 delta=-25 (val=5) hist 5 30 0 30 00000100:00001000:0.0:1788572605.645944:0:266399:0:(client.c:437:ptlrpc_at_adj_service()) The RPC service estimate for lustre-MDT0000-mdc-ffff9c3711155800 ptl 13 has changed from 30 to 5 00000100:00001000:0.0:1788572605.645946:0:266399:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=51 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572605.645952:0:266399:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572605.645953:0:266399:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572605.645955:0:266399:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572605.645957:0:266399:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.645960:0:266399:0:(client.c:1554:ptlrpc_save_versions()) Process entered 00000100:00000040:0.0:1788572605.645962:0:266399:0:(client.c:1561:ptlrpc_save_versions()) Client save versions [0x60000dadb/0x60000dadb] 00000100:00000001:0.0:1788572605.645963:0:266399:0:(client.c:1563:ptlrpc_save_versions()) Process leaving 00000100:00000001:0.0:1788572605.645966:0:266399:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.645968:0:266399:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000001:0.0:1788572605.645971:0:266399:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572605.645974:0:266399:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572605.645976:0:266399:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572605.645978:0:266399:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.645984:0:266399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.645991:0:266399:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.645994:0:266399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572605.645995:0:266399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572605.646000:0:266399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 3 fl Interpret:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572605.646008:0:266399:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job mv:5fa795d2-07a0-4260-8bbc-bb3cde572406:266399:1875440809992192:192.168.202.148@tcp:36:mv.0 00000100:00000001:0.0:1788572605.646012:0:266399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.646015:0:266399:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.646016:0:266399:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572605.646017:0:266399:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.646019:0:266399:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 3 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.646023:0:266399:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572605.646024:0:266399:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c37380b66c0. 00000100:00000001:0.0:1788572605.646026:0:266399:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572605.646027:0:266399:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.646030:0:266399:0:(mdc_reint.c:521:mdc_rename()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1788572605.646036:0:266399:0:(lmv_obd.c:3000:lmv_rename()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1788572605.646038:0:266399:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c373808f200. 00000100:00000001:0.0:1788572605.646040:0:266399:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.646042:0:266399:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 2 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.646046:0:266399:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.646051:0:266399:0:(namei.c:2493:ll_rename()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.646056:0:266399:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:0.0:1788572605.646059:0:266399:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:0.0:1788572605.646061:0:266399:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572605.646062:0:266399:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1f03de8 00000080:00002000:0.0:1788572605.646063:0:266399:0:(dcache.c:127:ll_ddelete()) deleting dentry f15 (ffff9c3703599798, parent ffff9c37034c5798, inode ffff9c37020e4b08) hashed, 00000080:00000001:0.0:1788572605.646065:0:266399:0:(dcache.c:130:ll_ddelete()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572605.646068:0:266399:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:0.0:1788572605.646070:0:266399:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:0.0:1788572605.646073:0:266399:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:0.0:1788572605.646073:0:266399:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1f03de8 00000080:00002000:0.0:1788572605.646074:0:266399:0:(dcache.c:127:ll_ddelete()) keeping dentry d3 (ffff9c37034c5798, parent ffff9c37034e2510, inode ffff9c37081f8908) hashed, 00000080:00000001:0.0:1788572605.646076:0:266399:0:(dcache.c:131:ll_ddelete()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572605.647937:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.647944:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.647946:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.647948:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.647952:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.647955:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.647970:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.647974:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.647978:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.647981:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746e280 00000400:00000200:0.0:1788572605.647985:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.647989:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52abe9 [1] + 224 00000800:00000001:0.0:1788572605.647994:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.648001:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.648003:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.648007:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.648018:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.648022:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572690 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572605.648031:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=456 offset=224 replen=512 req@ffff9c372d6b6a00 x1875440809992832/t0(0) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/512 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572605.648048:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.648053:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.648056:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572605.648909:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572605.648913:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572605.648917:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.648922:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572605.648925:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572605.648927:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572605.648930:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572605.648935:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5500 x1875440809992960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572690 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.648943:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572605.648946:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff9c371f1c5500 x1875440809992960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572690 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.648952:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.648954:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572605.650304:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.650313:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.650314:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.650316:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.650319:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.650323:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.650327:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.650329:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.650331:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.650334:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 400 into portal 10 MB=0x6a9b41746e300 00000400:00000200:0.0:1788572605.650343:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.650346:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 400/400 into md 0x52abf9 [1] + 224 00000800:00000001:0.0:1788572605.650350:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.650354:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.650355:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.650357:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.650360:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.650363:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5500 x1875440809992960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572690 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572605.650370:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=400 offset=224 replen=736 req@ffff9c371f1c5500 x1875440809992960/t0(0) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 376/736 e 0 to 0 dl 1788572690 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.650383:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.650387:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.650390:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.651290:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572605.651294:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.651295:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572605.651302:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440809993088/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 0 ref 1 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.651310:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572605.651312:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.651314:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572605.651316:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572605.651318:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572605.651321:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809993088:192.168.202.148@tcp:2:touch.0 00000100:00000001:0.0:1788572605.651325:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572605.651328:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572605.651330:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.651332:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572605.651336:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788572605.651338:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572605.651342:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572605.651352:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8580. 00000400:00000010:0.0:1788572605.651357:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788572605.651360:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809993088, portal 4 00000100:00000001:0.0:1788572605.651362:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.651363:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788572605.651369:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c371f1c5500 x1875440809993088/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572621 ref 2 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.651376:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572605.651379:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572605.651381:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788572605.651383:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 28, xid 1875440809993088, offset 0 00000400:00000200:0.0:1788572605.651387:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572605.651391:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.651395:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572605.651398:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572605.651401:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572605.651404:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572605.651406:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572605.651409:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540094:540094:256:4294967295] 192.168.202.148@tcp LPNI seq info [540059:540059:8:4294967295] 00000400:00000200:0.0:1788572605.651423:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.651430:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572605.651434:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572605.651439:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572605.651443:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572605.651448:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572605.651451:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 536 niov 1 nkiov 1 00000100:00000001:0.0:1788572605.651465:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.651469:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 169338 usecs (170 jiffies) 00000400:00000200:0.0:1788572605.651472:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572605.651473:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307744827 00000100:00000001:0.0:1788572605.651476:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.651486:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307744827 00000100:00000001:0.0:1788572605.651490:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.651492:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.651495:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.651498:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.651500:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572605.651506:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.651508:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572605.651511:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572605.651512:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.651513:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.651516:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.651527:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572605.651529:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572605.652672:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.652685:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.652689:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.652692:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.652700:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.652708:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.652716:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.652721:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.652725:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.652730:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 432 into portal 4 MB=0x6a9b41746e380 00000400:00000200:0.0:1788572605.652735:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.652740:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 432/432 into md 0x52ac09 [1] + 224 00000800:00000001:0.0:1788572605.652747:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.652755:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.652758:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.652762:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.652769:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.652776:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c5500 x1875440809993088/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572621 ref 1 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572605.652787:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=432 offset=224 replen=432 req@ffff9c371f1c5500 x1875440809993088/t0(0) o2->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 440/432 e 0 to 0 dl 1788572621 ref 1 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.652817:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.652824:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.652827:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572605.653019:0:266397:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572605.653022:0:266397:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653025:0:266397:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.653029:0:266397:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572605.653033:0:266397:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572605.653035:0:266397:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572605.653038:0:266397:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572605.653040:0:266397:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572605.653042:0:266397:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572605.653043:0:266397:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653045:0:266397:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653049:0:266397:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572605.653051:0:266397:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572605.653055:0:266397:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572605.653057:0:266397:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572605.653060:0:266397:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572605.653063:0:266397:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572605.653065:0:266397:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572605.653066:0:266397:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572605.653067:0:266397:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572605.653068:0:266397:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653070:0:266397:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653072:0:266397:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572605.653072:0:266397:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572605.653074:0:266397:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572605.653075:0:266397:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572605.653076:0:266397:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572605.653077:0:266397:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572605.653077:0:266397:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572605.653078:0:266397:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572605.653079:0:266397:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00020000:00000001:0.0:1788572605.653081:0:266397:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572605.653082:0:266397:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572605.653083:0:266397:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572605.653084:0:266397:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdff:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572605.653086:0:266397:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572605.653087:0:266397:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572605.653092:0:266397:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000080:00000001:0.0:1788572605.653093:0:266397:0:(lcommon_cl.c:108:cl_setattr_ost()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.653095:0:266397:0:(llite_lib.c:2656:ll_setattr_raw()) Process leaving 00000080:00000010:0.0:1788572605.653097:0:266397:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff9c370395b000. 00000020:00000001:0.0:1788572605.653103:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafc30 00000080:00200008:0.0:1788572605.653105:0:266397:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f13:[0x200002b17:0xdff:0x0](ffff9c37087063c8) current size 0, valid attrs 100201f0, mode 0, uid -1055292862, gid -1, new size 213440, atime 1788572605.000000000, mtime 1788572605.000000000, ctime 1788572605.000000000, rc 0 00000080:00000001:0.0:1788572605.653110:0:266397:0:(llite_lib.c:2688:ll_setattr_raw()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572605.653123:0:266397:0:(lov_object.c:2398:lov_read_and_clear_async_rc()) Process entered 00020000:00000002:0.0:1788572605.653125:0:266397:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000002:0.0:1788572605.653129:0:266397:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000001:0.0:1788572605.653131:0:266397:0:(lov_object.c:2449:lov_read_and_clear_async_rc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.653138:0:266397:0:(file.c:475:ll_file_release()) Process entered 00000020:00000001:0.0:1788572605.653140:0:266397:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffb95bc1eafdf0 00000080:00200008:0.0:1788572605.653142:0:266397:0:(file.c:479:ll_file_release()) START file f13:[0x200002b17:0xdff:0x0](ffff9c37087063c8), flags 104001 00000080:00000001:0.0:1788572605.653145:0:266397:0:(pcc.c:2472:pcc_file_release()) Process entered 00000080:00000001:0.0:1788572605.653147:0:266397:0:(pcc.c:2500:pcc_file_release()) Process leaving 00020000:00000001:0.0:1788572605.653148:0:266397:0:(lov_object.c:2398:lov_read_and_clear_async_rc()) Process entered 00020000:00000002:0.0:1788572605.653150:0:266397:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000002:0.0:1788572605.653152:0:266397:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3709594f80 00020000:00000001:0.0:1788572605.653154:0:266397:0:(lov_object.c:2449:lov_read_and_clear_async_rc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572605.653156:0:266397:0:(file.c:341:ll_md_close()) Process entered 00800000:00000001:0.0:1788572605.653158:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572605.653159:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572605.653160:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572605.653161:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572605.653163:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.653164:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdff:0x0] 00800000:00000001:0.0:1788572605.653165:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788572605.653167:0:266397:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788572605.653168:0:266397:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572605.653171:0:266397:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c3703a15600 count: 2 00010000:00000001:0.0:1788572605.653172:0:266397:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788572605.653174:0:266397:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.653175:0:266397:0:(lmv_obd.c:4142:lmv_lock_match()) Lock match for [0x200002b17:0xdff:0x0]: 0 00000080:00000001:0.0:1788572605.653181:0:266397:0:(file.c:297:ll_md_real_close()) Process entered 00000080:00000001:0.0:1788572605.653182:0:266397:0:(file.c:150:ll_close_inode_openhandle()) Process entered 00000080:00000010:0.0:1788572605.653186:0:266397:0:(file.c:158:ll_close_inode_openhandle()) kmalloced '(op_data)': 392 at ffff9c370395b000. 00000080:00000001:0.0:1788572605.653187:0:266397:0:(file.c:90:ll_prepare_close()) Process entered 00000080:00000001:0.0:1788572605.653190:0:266397:0:(file.c:128:ll_prepare_close()) Process leaving 00800000:00000001:0.0:1788572605.653192:0:266397:0:(lmv_obd.c:1670:lmv_close()) Process entered 00800000:00000001:0.0:1788572605.653193:0:266397:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1788572605.653194:0:266397:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1788572605.653195:0:266397:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1788572605.653196:0:266397:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1788572605.653197:0:266397:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.653198:0:266397:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200002b17:0xdff:0x0] 00800000:00000001:0.0:1788572605.653199:0:266397:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1788572605.653200:0:266397:0:(lmv_obd.c:1676:lmv_close()) CLOSE [0x200002b17:0xdff:0x0] 00000002:00000001:0.0:1788572605.653202:0:266397:0:(mdc_request.c:879:mdc_close()) Process entered 00000002:00000002:0.0:1788572605.653204:0:266397:0:(mdc_request.c:882:mdc_close()) lustre-MDT0000-mdc-ffff9c3711155800: [0x200002b17:0xdff:0x0] file closed with intent: 0 00000100:00000010:0.0:1788572605.653207:0:266397:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572605.653208:0:266397:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=8 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572605.653210:0:266397:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572605.653212:0:266397:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572605.653213:0:266397:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572605.653216:0:266397:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c37212bb400. 00000100:00000001:0.0:1788572605.653218:0:266397:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000002:00100000:0.0:1788572605.653226:0:266397:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788572605.653233:0:266397:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9c3711155800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788572605.653236:0:266397:0:(client.c:827:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9c372d6b5f80 x1875440809993344/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00000001:0.0:1788572605.653240:0:266397:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572605.653241:0:266397:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572605.653244:0:266397:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c371018a900. 00000100:00000001:0.0:1788572605.653245:0:266397:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359040026880 : -109714669524736 : ffff9c371018a900) 00000100:00000001:0.0:1788572605.653247:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.653248:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788572605.653252:0:266397:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572605.653254:0:266397:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.653255:0:266397:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572605.653256:0:266397:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572605.653258:0:266397:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809993344/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.653262:0:266397:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572605.653263:0:266397:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.653264:0:266397:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572605.653265:0:266397:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572605.653266:0:266397:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572605.653268:0:266397:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job touch:5fa795d2-07a0-4260-8bbc-bb3cde572406:266397:1875440809993344:192.168.202.148@tcp:35:touch.0 00000100:00000001:0.0:1788572605.653270:0:266397:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572605.653272:0:266397:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572605.653273:0:266397:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572605.653274:0:266397:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572605.653276:0:266397:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572605.653277:0:266397:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572605.653281:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572605.653283:0:266397:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572605.653285:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788572605.653288:0:266397:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1875440809993344, portal 10 00000100:00000001:0.0:1788572605.653289:0:266397:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572605.653290:0:266397:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572605.653293:0:266397:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=600 req@ffff9c372d6b5f80 x1875440809993344/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572621 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.653297:0:266397:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572605.653299:0:266397:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572605.653300:0:266397:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788572605.653301:0:266397:0:(niobuf.c:94:ptl_send_buf()) Sending 392 bytes to portal 23, xid 1875440809993344, offset 0 00000400:00000200:0.0:1788572605.653304:0:266397:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572605.653307:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.653310:0:266397:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572605.653315:0:266397:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572605.653317:0:266397:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572605.653319:0:266397:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572605.653321:0:266397:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572605.653323:0:266397:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540095:540095:256:4294967295] 192.168.202.148@tcp LPNI seq info [540060:540060:8:4294967295] 00000400:00000200:0.0:1788572605.653326:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.653330:0:266397:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572605.653333:0:266397:0:(socklnd_cb.c:931:ksocknal_send()) sending 392 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572605.653335:0:266397:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572605.653338:0:266397:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572605.653341:0:266397:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572605.653343:0:266397:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0800 type 1, nob 488 niov 1 nkiov 1 00000100:00000001:0.0:1788572605.653354:0:266397:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572605.653358:0:266397:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23500 usecs (24 jiffies) 00000400:00000200:0.0:1788572605.653361:0:266397:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572605.653363:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307744683 00000100:00000001:0.0:1788572605.653365:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.653367:0:266397:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307744683 00000100:00000001:0.0:1788572605.653369:0:266397:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307747550 00000100:00000001:0.0:1788572605.653371:0:266397:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572605.653373:0:266397:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572605.653374:0:266397:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572605.653376:0:266397:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1788572605.653378:0:266397:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c371018a900 going to sleep for 11 seconds 00000100:00000001:0.0:1788572605.653380:0:266397:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572605.653382:0:266397:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572605.654715:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.654725:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572605.654727:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.654730:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.654743:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.654756:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572605.654761:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.654764:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.654767:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.654770:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 456 into portal 10 MB=0x6a9b41746e480 00000400:00000200:0.0:1788572605.654774:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572605.654778:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 456/456 into md 0x52ac19 [1] + 224 00000800:00000001:0.0:1788572605.654783:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.654788:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572605.654790:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572605.654793:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572605.654796:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572605.654800:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5f80 x1875440809993344/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572621 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000040:0.0:1788572605.654808:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=600 mlen=456 offset=224 replen=888 req@ffff9c372d6b5f80 x1875440809993344/t0(0) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/888 e 0 to 0 dl 1788572621 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572605.654824:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572605.654829:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572605.654831:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572606.721699:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572606.721705:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572606.721708:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1788572606.721775:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788572606.721779:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572606.721786:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572606.721790:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572606.721791:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1788572607.318715:0:266824:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572607.682725:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572607.682728:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572607.682732:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1788572607.895724:0:266971:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572608.577690:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307747550, at 4307747584 00000100:00000010:0.0:1788572608.577703:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116269c0. 00000020:00000001:0.0:1788572608.705737:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572608.705742:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572608.705746:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572609.857723:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572609.857731:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572609.857740:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572609.857744:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572609.857747:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572609.857749:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572609.857753:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572609.857759:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0600. 00000100:00000001:0.0:1788572609.857765:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857773:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572609.857781:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572609.857784:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857791:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.857822:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857826:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572609.857829:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572609.857833:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572609.857836:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=9 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572609.857841:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572609.857844:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572609.857846:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572609.857852:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0500. 00000100:00000001:0.0:1788572609.857857:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857864:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572609.857884:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572609.857889:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857894:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.857908:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857910:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572609.857916:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572609.857921:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572609.857926:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=6 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572609.857929:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572609.857930:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572609.857931:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572609.857935:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0d00. 00000100:00000001:0.0:1788572609.857938:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857944:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572609.857952:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572609.857954:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857958:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.857963:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.857966:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572609.857971:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572609.857976:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7b80. 00000020:00000040:0.0:1788572609.857981:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572609.857986:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572609.857989:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572609.857993:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572609.857999:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0200. 00000100:00000001:0.0:1788572609.858010:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858020:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572609.858030:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572609.858032:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858036:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008757f27f] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858048:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858051:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572609.858057:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572609.858061:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7800. 00000020:00000040:0.0:1788572609.858073:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 00000100:00000001:0.0:1788572609.858079:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572609.858080:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572609.858081:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572609.858086:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0800. 00000100:00000001:0.0:1788572609.858090:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858098:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572609.858111:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572609.858116:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858124:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b4f2ac1] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858136:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858143:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13086) 00000100:00000001:0.0:1788572609.858154:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.858157:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572609.858160:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572609.858165:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858186:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572609.858189:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.858193:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572609.858197:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572609.858200:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572609.858205:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809993600:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572609.858213:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572609.858216:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572609.858219:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.858222:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572609.858226:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c373023c000. 02000000:00000001:0.0:1788572609.858230:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572609.858237:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572609.858242:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788572609.858247:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572609.858251:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809993600, portal 10 00000100:00000001:0.0:1788572609.858254:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572609.858258:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000040:0.0:1788572609.858265:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858278:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572609.858283:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572609.858287:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572609.858292:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809993600, offset 0 00000400:00000200:0.0:1788572609.858299:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572609.858306:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.858313:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572609.858317:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572609.858320:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572609.858324:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572609.858330:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572609.858338:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540097:540097:255:4294967295] 192.168.202.148@tcp LPNI seq info [540062:540062:7:4294967295] 00000400:00000200:0.0:1788572609.858341:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.858345:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572609.858349:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572609.858354:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572609.858362:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572609.858371:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572609.858376:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572609.858381:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858385:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23492 usecs (24 jiffies) 00000400:00000200:0.0:1788572609.858399:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572609.858404:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307748888 00000100:00000001:0.0:1788572609.858409:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307752670 00000100:00000001:0.0:1788572609.858413:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307748888 00000100:00000001:0.0:1788572609.858417:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307752670 00000100:00000001:0.0:1788572609.858420:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572609.858423:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858427:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572609.858433:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858442:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572609.858443:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.858448:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572609.858449:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572609.858453:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572609.858456:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809993856:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572609.858463:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572609.858466:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572609.858468:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.858497:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572609.858503:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c373023da00. 02000000:00000001:0.0:1788572609.858509:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572609.858525:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572609.858531:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788572609.858538:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572609.858544:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809993856, portal 4 00000100:00000001:0.0:1788572609.858550:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572609.858554:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531993984 : -109714177557632 : ffff9c372d6b7b80) 00000100:00000040:0.0:1788572609.858567:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858575:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572609.858580:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572609.858585:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572609.858589:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809993856, offset 0 00000400:00000200:0.0:1788572609.858596:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572609.858605:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.858611:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572609.858614:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572609.858616:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572609.858617:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572609.858625:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572609.858629:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540099:540099:253:4294967295] 192.168.202.148@tcp LPNI seq info [540064:540064:5:4294967295] 00000400:00000200:0.0:1788572609.858633:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.858664:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572609.858670:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572609.858677:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572609.858686:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572609.858695:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572609.858700:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572609.858705:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.858714:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 160462 usecs (161 jiffies) 00000400:00000200:0.0:1788572609.858720:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572609.858723:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307749026 00000100:00000001:0.0:1788572609.858727:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307752670 00000100:00000001:0.0:1788572609.858731:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307749026 00000100:00000001:0.0:1788572609.858735:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307752670 00000100:00000001:0.0:1788572609.858740:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572609.858744:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858749:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858753:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.858758:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572609.858761:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572609.858766:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.858768:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572609.858770:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858773:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572609.858809:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572609.858812:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572609.858816:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.858820:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572609.858823:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3880 00000400:00000010:0.0:1788572609.858825:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3880. 00000100:00000001:0.0:1788572609.858827:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572609.858830:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858837:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.858839:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858843:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858844:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572609.858850:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572609.858852:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572609.858853:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.858859:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572609.858861:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31198 00000400:00000010:0.0:1788572609.858862:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31198. 00000100:00000001:0.0:1788572609.858863:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572609.858866:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858870:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.858871:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858875:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858876:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572609.858880:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572609.858881:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572609.858882:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.858884:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572609.858885:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3a18 00000400:00000010:0.0:1788572609.858886:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3a18. 00000100:00000001:0.0:1788572609.858887:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572609.858889:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858893:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.858894:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858898:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858898:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572609.858901:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572609.858902:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572609.858904:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.858905:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572609.858906:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572609.858907:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572609.858908:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572609.858913:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858917:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.858918:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858922:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858922:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572609.858963:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572609.858965:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572609.858967:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.858968:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572609.858970:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3660 00000400:00000010:0.0:1788572609.858972:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3660. 00000100:00000001:0.0:1788572609.858974:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572609.858977:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858983:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.858986:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.858991:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.858993:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572609.860547:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860556:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572609.860559:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860560:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860564:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860568:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572609.860572:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860575:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860578:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860580:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b41746e500 00000400:00000200:0.0:1788572609.860583:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860592:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac29 [1] + 224 00000800:00000001:0.0:1788572609.860597:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860599:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860602:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.860604:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572609.860607:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.860611:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572609.860617:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5880 x1875440809993472/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.860633:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572609.860660:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860673:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572609.860678:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860682:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860688:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860694:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572609.860699:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860706:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860712:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860719:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b41746e580 00000400:00000200:0.0:1788572609.860727:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860732:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac39 [1] + 224 00000800:00000001:0.0:1788572609.860738:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860744:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860747:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.860755:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572609.860758:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.860763:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572609.860777:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.860814:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572609.860819:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860823:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572609.860826:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860829:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860834:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860842:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572609.860849:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860854:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860858:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860863:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b41746e600 00000400:00000200:0.0:1788572609.860867:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.860873:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac41 [1] + 224 00000800:00000001:0.0:1788572609.860879:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860883:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.860886:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.860890:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572609.860894:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.860898:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572609.860906:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5180 x1875440809993728/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.860917:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572609.860920:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.860923:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.860932:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.860935:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572609.860939:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572609.860942:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572609.860944:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8210. 00000400:00000200:0.0:1788572609.860947:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788572609.860954:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788572609.860957:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.860959:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572609.860964:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.860971:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572609.860974:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.860979:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572609.860982:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572609.860986:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572609.860989:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.860993:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572609.861000:0:2404:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=23 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572609.861004:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=55 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572609.861008:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572609.861011:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572609.861015:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572609.861019:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861024:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572609.861026:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: skip recheck: last_committed 25769859811 00000100:00000001:0.0:1788572609.861030:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572609.861032:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.861039:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861049:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572609.861052:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572609.861054:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572609.861060:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572609.861066:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809993600:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572609.861069:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861076:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.861081:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440809993600/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572694 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861095:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572609.861098:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572609.861103:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c373023c000. 02000000:00000001:0.0:1788572609.861109:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572609.861112:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572609.861116:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=8 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572609.861120:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572609.861124:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0500. 02000000:00000001:0.0:1788572609.861129:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572609.861130:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572609.861132:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788572609.861135:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572609.861136:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861139:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861142:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.861143:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572609.861145:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861150:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572609.861159:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861164:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572609.861168:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861171:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.861176:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861182:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572609.861188:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861191:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861196:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861201:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b41746e680 00000400:00000200:0.0:1788572609.861205:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861209:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac59 [1] + 224 00000800:00000001:0.0:1788572609.861214:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861217:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.861226:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.861230:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572609.861234:0:2396:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.861239:0:2396:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572609.861265:0:2396:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861276:0:2396:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572609.861281:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861284:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861292:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.861295:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572609.861299:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572609.861301:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572609.861304:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8f78. 00000400:00000200:0.0:1788572609.861309:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572609.861312:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572609.861315:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.861321:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572609.861335:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861344:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572609.861346:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861347:0:2404:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572609.861350:0:2404:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572609.861351:0:2404:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572609.861354:0:2404:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572609.861357:0:2404:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572609.861363:0:2404:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=23 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572609.861366:0:2404:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=74 v=5 (1 1 2 0) 00000100:00000001:0.0:1788572609.861371:0:2404:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572609.861375:0:2404:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572609.861377:0:2404:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572609.861384:0:2404:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861389:0:2404:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572609.861391:0:2404:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727883 00000100:00000001:0.0:1788572609.861394:0:2404:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572609.861398:0:2404:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572609.861403:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861413:0:2404:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572609.861415:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572609.861417:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572609.861422:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572609.861431:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809993856:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572609.861439:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861442:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572609.861448:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7b80 x1875440809993856/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861457:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572609.861460:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572609.861462:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c373023da00. 02000000:00000001:0.0:1788572609.861466:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572609.861469:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572609.861486:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572609.861491:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572609.861493:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0200. 02000000:00000001:0.0:1788572609.861497:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572609.861499:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572609.861502:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7b80. 00000100:00000001:0.0:1788572609.861505:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572609.861508:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861513:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572609.861516:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572609.861523:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572609.861528:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572609.861530:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572609.861537:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861541:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572609.861542:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861543:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.861546:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861549:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572609.861553:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861555:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861557:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861558:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b41746e700 00000400:00000200:0.0:1788572609.861560:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572609.861562:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac61 [1] + 224 00000800:00000001:0.0:1788572609.861565:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861573:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572609.861575:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572609.861577:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572609.861579:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572609.861581:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572609.861586:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7800 x1875440809993984/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572609.861597:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572609.861599:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572609.861601:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572611.092549:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.092573:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572611.092578:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.092580:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572611.092589:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.092596:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572611.092605:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.092623:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.092629:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.092633:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 328 into portal 15 MB=0x6a9b414932600 00000400:00000200:0.0:1788572611.092688:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index f from 12345-192.168.202.148@tcp of length 328/328 into md 0x525bc1 [2] + 984 00000800:00000001:0.0:1788572611.092696:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.092704:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572611.092707:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572611.092712:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572611.092719:0:2396:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:0.0:1788572611.092722:0:2396:0:(events.c:300:request_in_callback()) event type 2, status 0, service ldlm_cbd 00000100:00000010:0.0:1788572611.092727:0:2396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7b80. 00000100:00000040:0.0:1788572611.092731:0:2396:0:(events.c:341:request_in_callback()) incoming req@ffff9c372d6b7b80 x1875440764659200 msgsize 328 00000100:00100000:0.0:1788572611.092736:0:2396:0:(events.c:344:request_in_callback()) peer: 12345-192.168.202.148@tcp (source: 12345-192.168.202.148@tcp) 00000100:00000001:0.0:1788572611.092748:0:2396:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:0.0:1788572611.092755:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.092758:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572611.093108:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572611.093115:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788572611.093123:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572611.093131:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572611.093137:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f35d8 00000400:00000010:0.0:1788572611.093142:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f35d8. 00000100:00000001:0.0:1788572611.093147:0:2395:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:0.0:1788572611.093150:0:2395:0:(events.c:402:reply_out_callback()) Process leaving 00000800:00000200:0.0:1788572611.093552:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572611.093556:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498000. 00000400:00000200:0.0:1788572611.093560:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572611.093564:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572611.093567:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c12a8 00000400:00000010:0.0:1788572611.093569:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c12a8. 00000100:00000001:0.0:1788572611.093572:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572611.093577:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809994112/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572627 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572611.093586:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572611.093595:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809994112/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572627 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572611.093603:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572611.093605:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572611.094503:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.094510:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572611.094512:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.094515:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572611.094518:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.094522:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572611.094527:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.094529:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.094532:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.094535:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41746e780 00000400:00000200:0.0:1788572611.094539:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572611.094543:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac81 [1] + 224 00000800:00000001:0.0:1788572611.094547:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.094551:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572611.094553:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572611.094557:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572611.094560:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572611.094564:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed69880 x1875440809994112/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572627 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572611.094574:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed69880 x1875440809994112/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572627 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572611.094589:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572611.094594:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572611.094596:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572611.777810:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572611.777815:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572611.777818:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572612.802915:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572612.802924:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572612.802926:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572613.697688:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307752670, at 4307752704 00000100:00000010:0.0:1788572613.697703:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00000040:0.0:1788572614.977759:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572614.977768:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572614.977777:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572614.977783:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572614.977787:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572614.977790:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572614.977795:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572614.977802:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0200. 00000100:00000001:0.0:1788572614.977808:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977819:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 874b6a47-e66d-4b10-8108-059b044e1170->MGS req@ffff9c372d6b5500 x1875440809994240/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572614.977829:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572614.977833:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977840:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5500 x1875440809994240/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.977873:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977876:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572614.977880:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572614.977883:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572614.977886:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=9 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572614.977888:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572614.977890:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572614.977891:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572614.977895:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0500. 00000100:00000001:0.0:1788572614.977899:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977904:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572614.977934:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572614.977939:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977948:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.977965:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.977969:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572614.977975:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572614.977979:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572614.977985:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572614.977989:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572614.977990:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572614.977994:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572614.978008:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0f00. 00000100:00000001:0.0:1788572614.978014:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978019:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID req@ffff9c372d6b4380 x1875440809994496/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572614.978032:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572614.978036:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978045:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4380 x1875440809994496/t0(0) o400->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978064:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978068:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572614.978075:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572614.978081:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7800. 00000020:00000040:0.0:1788572614.978085:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572614.978089:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572614.978092:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572614.978095:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572614.978100:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0900. 00000100:00000001:0.0:1788572614.978112:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978121:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572614.978149:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572614.978153:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978162:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b4f2ac1] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978171:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978174:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572614.978180:0:234251:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572614.978184:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572614.978187:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 00000100:00000001:0.0:1788572614.978192:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572614.978194:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572614.978198:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572614.978203:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0e00. 00000100:00000001:0.0:1788572614.978207:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978213:0:234251:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID req@ffff9c372d6b5180 x1875440809994752/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572614.978226:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572614.978230:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978237:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5180 x1875440809994752/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978248:0:234251:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978256:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13091) 00000100:00000001:0.0:1788572614.978311:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.978315:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572614.978318:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572614.978326:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978373:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572614.978377:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.978383:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572614.978396:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572614.978405:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572614.978413:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809994368:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572614.978421:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572614.978425:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572614.978429:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.978433:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572614.978439:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de6800. 02000000:00000001:0.0:1788572614.978444:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572614.978452:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572614.978457:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788572614.978463:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572614.978468:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809994368, portal 10 00000100:00000001:0.0:1788572614.978473:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572614.978476:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000040:0.0:1788572614.978486:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978500:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572614.978507:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572614.978512:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788572614.978517:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809994368, offset 0 00000400:00000200:0.0:1788572614.978527:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572614.978536:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.978545:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572614.978551:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572614.978557:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572614.978563:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572614.978567:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572614.978582:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540105:540105:254:4294967295] 192.168.202.148@tcp LPNI seq info [540070:540070:6:4294967295] 00000400:00000200:0.0:1788572614.978592:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.978607:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572614.978613:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572614.978618:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572614.978625:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572614.978633:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572614.978655:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572614.978660:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978663:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23485 usecs (24 jiffies) 00000400:00000200:0.0:1788572614.978671:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572614.978675:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307754009 00000100:00000001:0.0:1788572614.978680:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307757790 00000100:00000001:0.0:1788572614.978686:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307754009 00000100:00000001:0.0:1788572614.978692:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307757790 00000100:00000001:0.0:1788572614.978697:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572614.978702:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.978706:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572614.978714:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978732:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572614.978735:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.978740:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572614.978743:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572614.978749:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572614.978757:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809994624:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572614.978764:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572614.978768:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572614.978771:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.978781:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572614.978786:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de7000. 02000000:00000001:0.0:1788572614.978790:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572614.978794:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572614.978797:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788572614.978801:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572614.978804:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809994624, portal 4 00000100:00000001:0.0:1788572614.978806:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572614.978807:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531993088 : -109714177558528 : ffff9c372d6b7800) 00000100:00000040:0.0:1788572614.978814:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.978821:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572614.978823:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572614.978827:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572614.978831:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809994624, offset 0 00000400:00000200:0.0:1788572614.978838:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572614.978846:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.978854:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572614.978858:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572614.978860:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572614.978863:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572614.978865:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572614.978868:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540107:540107:252:4294967295] 192.168.202.148@tcp LPNI seq info [540072:540072:4:4294967295] 00000400:00000200:0.0:1788572614.978873:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.978879:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572614.978883:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572614.978887:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788572614.978895:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572614.978902:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572614.978906:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572614.978913:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.978921:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 160148 usecs (161 jiffies) 00000400:00000200:0.0:1788572614.978929:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572614.978933:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307754146 00000100:00000001:0.0:1788572614.978939:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307757790 00000100:00000001:0.0:1788572614.978942:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307754146 00000100:00000001:0.0:1788572614.978947:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307757790 00000100:00000001:0.0:1788572614.978950:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572614.978952:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.978955:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.978958:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.978964:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572614.978968:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1788572614.978973:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.978975:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572614.978979:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.978984:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572614.980461:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.980472:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572614.980475:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.980477:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.980482:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.980487:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572614.980493:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.980497:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.980500:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.980503:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 25 MB=0x6a9b41746e800 00000400:00000200:0.0:1788572614.980508:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.980513:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ac89 [1] + 224 00000800:00000001:0.0:1788572614.980518:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.980523:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.980526:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572614.980529:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572614.980534:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.980545:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5500 x1875440809994240/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572614.980555:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5500 x1875440809994240/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.980571:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572614.980577:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.980580:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981072:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.981076:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572614.981089:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572614.981092:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572614.981095:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8f78. 00000400:00000200:0.0:1788572614.981106:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572614.981110:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572614.981115:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.981123:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572614.981144:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981163:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572614.981166:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981175:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572614.981179:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572614.981181:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572614.981183:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.981187:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572614.981193:0:2403:0:(import.c:2195:obd_at_measure()) add 60 to ffff9c3707cce460 time=28 v=60 (60 61 61 61) 00000100:00001000:0.0:1788572614.981201:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707cce418 time=60 v=5 (1 1 1 1) 00000100:00000001:0.0:1788572614.981204:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572614.981207:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572614.981210:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572614.981215:0:2403:0:(ldlm_request.c:1703:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981219:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00100000:0.0:1788572614.981223:0:2403:0:(client.c:3133:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: committing for last_committed 25769859816 gen 1 00000100:00000040:0.0:1788572614.981238:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 25769859816) req@ffff9c372d6b5c00 x1875440809992320/t25769859812(25769859812) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981253:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981260:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5c00 x1875440809992320/t25769859812(25769859812) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981271:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981284:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981287:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ec800. 02000000:00000001:0.0:1788572614.981291:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981295:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981298:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=8 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981302:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981305:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3711c20200. 02000000:00000001:0.0:1788572614.981314:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981317:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981320:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5c00. 00000100:00000001:0.0:1788572614.981323:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981325:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981333:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 25769859816) req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981348:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572614.981351:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000001:0.0:1788572614.981358:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981363:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 2 fl Complete:RMQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981372:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981375:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981383:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440809993344/t25769859816(25769859816) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572621 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981393:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572614.981395:0:2403:0:(client.c:2955:obd_mod_free()) kfreed 'mod': 32 at ffff9c3703a9b960. 00000100:00000001:0.0:1788572614.981403:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981407:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440809990912/t25769859813(25769859813) o101->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 576/664 e 0 to 0 dl 1788572690 ref 1 fl Complete:RMQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981413:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788572614.981415:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981416:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=7 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981418:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981420:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788572614.981422:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981424:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981425:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788572614.981429:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981430:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981434:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 25769859816) req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981441:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981445:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440809992192/t25769859814(25769859814) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:13/10 lens 504/456 e 0 to 0 dl 1788572652 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981450:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981452:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981453:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37051a5800. 02000000:00000001:0.0:1788572614.981456:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981457:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981458:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=6 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981459:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981461:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c373808e800. 02000000:00000001:0.0:1788572614.981463:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981464:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981465:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788572614.981467:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981469:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981473:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 25769859816) req@ffff9c372d6b6a00 x1875440809992832/t25769859815(25769859815) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981488:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981491:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440809992832/t25769859815(25769859815) o36->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 488/456 e 0 to 0 dl 1788572690 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981497:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981498:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981499:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572614.981501:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981502:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981503:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981505:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981506:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3713f8a800. 02000000:00000001:0.0:1788572614.981508:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981509:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981510:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788572614.981512:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981513:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981516:0:2403:0:(client.c:3189:ptlrpc_free_committed()) @@@ commit (last_committed 25769859816) req@ffff9c372d6b5f80 x1875440809993344/t25769859816(25769859816) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572621 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981522:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981525:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440809993344/t25769859816(25769859816) o35->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:23/10 lens 392/456 e 0 to 0 dl 1788572621 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572614.981531:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981533:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981534:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff9c37109ef000. 02000000:00000001:0.0:1788572614.981536:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981537:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981538:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981540:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981541:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c37212bb400. 02000000:00000001:0.0:1788572614.981543:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981544:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981546:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788572614.981554:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981555:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981558:0:2403:0:(client.c:3196:ptlrpc_free_committed()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788572614.981560:0:2403:0:(client.c:3243:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9c3711155800: normal: skip=1 replay=5/5 committed=0/0 00000100:00000001:0.0:1788572614.981562:0:2403:0:(client.c:3244:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572614.981564:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.981569:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981576:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572614.981578:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572614.981579:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981583:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572614.981590:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809994368:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572614.981595:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981597:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981599:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440809994368/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572699 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981606:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981607:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981609:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3704de6800. 02000000:00000001:0.0:1788572614.981611:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981612:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981614:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981615:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981617:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0500. 02000000:00000001:0.0:1788572614.981619:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981620:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981621:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788572614.981623:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981625:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981627:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981631:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.981633:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572614.981634:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981658:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572614.981666:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981671:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572614.981673:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981675:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.981678:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981683:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572614.981687:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981690:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981693:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981697:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b41746ea00 00000400:00000200:0.0:1788572614.981700:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981704:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52acb9 [1] + 224 00000800:00000001:0.0:1788572614.981708:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981710:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.981712:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572614.981715:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572614.981719:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.981722:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b5180 x1875440809994752/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572614.981731:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b5180 x1875440809994752/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981738:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572614.981742:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981744:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572614.981746:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981748:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.981750:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981754:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572614.981758:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981766:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981769:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981772:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 4 MB=0x6a9b41746e980 00000400:00000200:0.0:1788572614.981775:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.981778:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52acc9 [1] + 224 00000800:00000001:0.0:1788572614.981782:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981787:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.981789:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572614.981793:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572614.981796:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.981800:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572614.981808:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981819:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572614.981823:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.981825:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981833:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.981834:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1788572614.981837:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572614.981838:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572614.981840:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8210. 00000400:00000200:0.0:1788572614.981843:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31770 00000400:00000010:0.0:1788572614.981845:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31770. 00000100:00000001:0.0:1788572614.981847:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.981851:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572614.981859:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981866:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572614.981867:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981869:0:2403:0:(client.c:1600:after_reply()) Process entered 02000000:00000001:0.0:1788572614.981876:0:2403:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1788572614.981877:0:2403:0:(pack_generic.c:606:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1788572614.981879:0:2403:0:(pack_generic.c:624:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572614.981881:0:2403:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1788572614.981885:0:2403:0:(import.c:2195:obd_at_measure()) add 5 to ffff9c3707ccac60 time=28 v=5 (5 5 5 5) 00000100:00001000:0.0:1788572614.981889:0:2403:0:(import.c:2195:obd_at_measure()) add 1 to ffff9c3707ccac18 time=79 v=5 (1 1 2 0) 00000100:00000001:0.0:1788572614.981891:0:2403:0:(client.c:1502:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1788572614.981893:0:2403:0:(client.c:1540:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572614.981895:0:2403:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1788572614.981897:0:2403:0:(ldlm_request.c:1703:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981899:0:2403:0:(client.c:3117:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1788572614.981901:0:2403:0:(client.c:3129:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9c3711155800: skip recheck: last_committed 38654727883 00000100:00000001:0.0:1788572614.981903:0:2403:0:(client.c:3130:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1788572614.981904:0:2403:0:(client.c:1805:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572614.981908:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981915:0:2403:0:(client.c:2362:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572614.981917:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572614.981919:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572614.981922:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572614.981930:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809994624:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572614.981934:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981936:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.981939:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7800 x1875440809994624/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572630 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.981945:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572614.981947:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572614.981949:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3704de7000. 02000000:00000001:0.0:1788572614.981952:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572614.981954:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572614.981955:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572614.981957:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572614.981962:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0900. 02000000:00000001:0.0:1788572614.981965:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572614.981966:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572614.981967:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7800. 00000100:00000001:0.0:1788572614.981970:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572614.981971:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981973:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572614.981978:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572614.981979:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.981981:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572614.981983:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572614.986202:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572614.986207:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498300. 00000400:00000200:0.0:1788572614.986212:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572614.986218:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572614.986222:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1e58 00000400:00000010:0.0:1788572614.986224:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1e58. 00000100:00000001:0.0:1788572614.986227:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572614.986232:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c4380 x1875440809994880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.986240:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572614.986244:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440809994880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572624 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.986251:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572614.986252:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572614.989885:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.989907:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572614.989911:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.989912:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.989918:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.989922:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572614.989928:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.989931:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.989934:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.989938:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 10 MB=0x6a9b41746ea80 00000400:00000200:0.0:1788572614.989950:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572614.989955:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.148@tcp of length 224/224 into md 0x52acd9 [1] + 0 00000800:00000001:0.0:1788572614.989960:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.989964:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572614.989967:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572614.989969:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572614.989974:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572614.989980:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371f1c4380 x1875440809994880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572624 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572614.989987:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=0 replen=544 req@ffff9c371f1c4380 x1875440809994880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572624 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572614.990002:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572614.990008:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572614.990009:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572615.059997:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.060013:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572615.060017:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.060019:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572615.060028:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.060037:0:2396:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572615.060045:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.060049:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.060054:0:2396:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.060058:0:2396:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 328 into portal 15 MB=0x6a9b414934480 00000400:00000200:0.0:1788572615.060066:0:2396:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index f from 12345-192.168.202.148@tcp of length 328/328 into md 0x525bc1 [2] + 1312 00000800:00000001:0.0:1788572615.060072:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.060078:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572615.060082:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572615.060086:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572615.060093:0:2396:0:(events.c:290:request_in_callback()) Process entered 00000100:00000200:0.0:1788572615.060096:0:2396:0:(events.c:300:request_in_callback()) event type 2, status 0, service ldlm_cbd 00000100:00000010:0.0:1788572615.060101:0:2396:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7800. 00000100:00000040:0.0:1788572615.060115:0:2396:0:(events.c:341:request_in_callback()) incoming req@ffff9c372d6b7800 x1875440764667008 msgsize 328 00000100:00100000:0.0:1788572615.060120:0:2396:0:(events.c:344:request_in_callback()) peer: 12345-192.168.202.148@tcp (source: 12345-192.168.202.148@tcp) 00000100:00000001:0.0:1788572615.060137:0:2396:0:(events.c:377:request_in_callback()) Process leaving 00000800:00000001:0.0:1788572615.060144:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.060147:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1788572615.060628:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572615.060634:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572615.060681:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572615.060688:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572615.060694:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f34c8 00000400:00000010:0.0:1788572615.060698:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f34c8. 00000100:00000001:0.0:1788572615.060704:0:2395:0:(events.c:390:reply_out_callback()) Process entered 00000100:00000001:0.0:1788572615.060706:0:2395:0:(events.c:402:reply_out_callback()) Process leaving 00000800:00000200:0.0:1788572615.061202:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572615.061206:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498f00. 00000400:00000200:0.0:1788572615.061210:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572615.061214:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572615.061217:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1bb0 00000400:00000010:0.0:1788572615.061219:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1bb0. 00000100:00000001:0.0:1788572615.061222:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572615.061226:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b100 x1875440809995008/t0(0) o103->MGC192.168.202.148@tcp@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572631 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572615.061236:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572615.061240:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440809995008/t0(0) o103->MGC192.168.202.148@tcp@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572631 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572615.061248:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572615.061249:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:0.0:1788572615.063449:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.063459:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572615.063461:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.063462:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572615.063467:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.063471:0:2395:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp) <- 192.168.202.148@tcp : PUT - for me 00000400:00000200:0.0:1788572615.063476:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.063488:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.063490:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.063493:0:2395:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.148@tcp of length 224 into portal 18 MB=0x6a9b41746eb00 00000400:00000200:0.0:1788572615.063496:0:2395:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572615.063500:0:2395:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.202.148@tcp of length 224/224 into md 0x52ace9 [1] + 224 00000800:00000001:0.0:1788572615.063504:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.063508:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788572615.063510:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572615.063513:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000100:00000001:0.0:1788572615.063517:0:2395:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572615.063521:0:2395:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff9c371ed6b100 x1875440809995008/t0(0) o103->MGC192.168.202.148@tcp@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572631 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788572615.063528:0:2395:0:(events.c:151:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff9c371ed6b100 x1875440809995008/t0(0) o103->MGC192.168.202.148@tcp@192.168.202.148@tcp:17/18 lens 328/224 e 0 to 0 dl 1788572631 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572615.063540:0:2395:0:(events.c:165:reply_in_callback()) Process leaving 00000800:00000001:0.0:1788572615.063544:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572615.063546:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572615.873841:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572615.873844:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572615.873846:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572616.897760:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572616.897763:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572616.897765:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788572617.025728:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788572617.025735:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788572617.025740:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788572617.025742:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788572617.025743:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788572617.025745:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788572617.025748:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788572617.025750:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788572617.025752:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788572617.025755:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788572617.025757:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788572617.025779:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572617.025782:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572617.025787:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572617.025789:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572617.025794:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572617.025799:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7000. 00000100:00000001:0.0:1788572617.025804:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572617.025807:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572617.025809:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572617.025815:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46900. 00010000:00000001:0.0:1788572617.025818:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572617.025820:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788572617.025823:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572617.025825:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x14415860931689d2 to hash 00000020:00000001:0.0:1788572617.025826:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572617.025827:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788572617.025829:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788572617.025831:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572617.025834:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572617.025836:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x14415860931689d2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572617.025843:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x14415860931689d2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572617.025851:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x14415860931689d2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572617.025854:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572617.025855:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572617.025860:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61540. 00000100:00000001:0.0:1788572617.025861:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610048 : -109714523941568 : ffff9c3718c61540) 00000100:00000001:0.0:1788572617.025863:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572617.025863:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788572617.025870:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572617.025873:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572617.025875:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572617.025876:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572617.025884:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440809995136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572617.025890:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572617.025891:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572617.025893:0:237086:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572617.025896:0:237086:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572617.025897:0:237086:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572617.025899:0:237086:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440809995136:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788572617.025902:0:237086:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572617.025906:0:237086:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572617.025907:0:237086:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572617.025909:0:237086:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572617.025913:0:237086:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572617.025914:0:237086:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572617.025920:0:237086:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572617.025922:0:237086:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788572617.025928:0:237086:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572617.025933:0:237086:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809995136, portal 25 00000100:00000001:0.0:1788572617.025934:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572617.025935:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000040:0.0:1788572617.025938:0:237086:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4a80 x1875440809995136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 1788572633 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572617.025943:0:237086:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572617.025945:0:237086:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572617.025946:0:237086:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572617.025948:0:237086:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1875440809995136, offset 0 00000400:00000200:0.0:1788572617.025951:0:237086:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572617.025957:0:237086:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572617.025963:0:237086:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572617.025970:0:237086:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572617.025973:0:237086:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572617.025975:0:237086:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572617.025977:0:237086:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572617.025980:0:237086:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540111:540111:256:4294967295] 192.168.202.148@tcp LPNI seq info [540076:540076:8:4294967295] 00000400:00000200:0.0:1788572617.025983:0:237086:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572617.025989:0:237086:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572617.025993:0:237086:0:(socklnd_cb.c:931:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572617.025997:0:237086:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572617.026002:0:237086:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572617.026007:0:237086:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572617.026010:0:237086:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1788572617.026026:0:237086:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572617.026028:0:237086:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572617.026031:0:237086:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307759838 00000100:00000001:0.0:1788572617.026036:0:237086:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4307757873 00000100:00000001:0.0:1788572617.026039:0:237086:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307759838 00000100:00000001:0.0:1788572617.026042:0:237086:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4307757873 00000100:00000001:0.0:1788572617.026043:0:237086:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572617.026045:0:237086:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572617.026047:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572617.026048:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1788572617.026050:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61540 going to sleep for 11 seconds 00000100:00000001:0.0:1788572617.026052:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572617.026054:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572620.098243:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572620.098252:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498600. 00000400:00000200:0.0:1788572620.098259:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572620.098272:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572620.098281:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1908 00000400:00000010:0.0:1788572620.098288:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1908. 00000100:00000001:0.0:1788572620.098301:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572620.098311:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68a80 x1875440809995264/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098337:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572620.098345:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440809995264/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572636 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098373:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572620.098379:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572620.098618:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572620.098623:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572620.098629:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572620.098658:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572620.098661:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3aa0 00000400:00000010:0.0:1788572620.098665:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3aa0. 00000100:00000001:0.0:1788572620.098668:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572620.098673:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68700 x1875440809995520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098696:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572620.098701:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440809995520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572635 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098720:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572620.098724:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572620.098739:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572620.098745:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498e00. 00000400:00000200:0.0:1788572620.098749:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572620.098758:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572620.098762:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1dd0 00000400:00000010:0.0:1788572620.098764:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1dd0. 00000100:00000001:0.0:1788572620.098771:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572620.098779:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69500 x1875440809995392/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098793:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572620.098805:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440809995392/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572705 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098819:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572620.098825:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572620.098838:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572620.098844:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37185d5f00. 00000400:00000200:0.0:1788572620.098851:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572620.098862:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572620.098867:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1880 00000400:00000010:0.0:1788572620.098879:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1880. 00000100:00000001:0.0:1788572620.098885:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572620.098892:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69c00 x1875440809995648/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098903:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572620.098908:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440809995648/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572636 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572620.098918:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572620.098921:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788572622.017863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572622.017866:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572622.017869:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572623.937762:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307762910, at 4307762944 00000100:00000010:0.0:1788572623.937780:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000800:00000200:0.0:1788572625.218427:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572625.218436:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498700. 00000400:00000200:0.0:1788572625.218446:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572625.218459:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572625.218469:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c13b8 00000400:00000010:0.0:1788572625.218475:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c13b8. 00000100:00000001:0.0:1788572625.218484:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572625.218497:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed68e00 x1875440809995904/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.218519:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572625.218536:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440809995904/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.218557:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572625.218561:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572625.219563:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572625.219569:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498200. 00000400:00000200:0.0:1788572625.219576:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572625.219584:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572625.219591:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1cc0 00000400:00000010:0.0:1788572625.219596:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1cc0. 00000100:00000001:0.0:1788572625.219603:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572625.219612:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69880 x1875440809996160/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.219622:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572625.219626:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440809996160/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.219633:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572625.219678:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572625.219873:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572625.219880:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696ff00. 00000400:00000200:0.0:1788572625.219886:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572625.219893:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572625.219896:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1198 00000400:00000010:0.0:1788572625.219899:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1198. 00000100:00000001:0.0:1788572625.219903:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572625.219908:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6a680 x1875440809996288/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.219916:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572625.219920:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440809996288/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572641 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572625.219927:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572625.219929:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788572627.137840:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572627.137844:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572627.137847:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572629.057748:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307768030, at 4307768064 00000100:00000010:0.0:1788572629.057765:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215000. 00000100:00000001:0.0:1788572630.337989:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572630.338004:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572630.338008:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572630.338012:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572630.338017:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572630.338021:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.338025:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572630.339896:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572630.339902:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f000. 00000400:00000200:0.0:1788572630.339909:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572630.339915:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572630.339922:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1990 00000400:00000010:0.0:1788572630.339925:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1990. 00000100:00000001:0.0:1788572630.339930:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572630.339935:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b800 x1875440809996544/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572715 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.339946:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572630.339950:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440809996544/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572715 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.339957:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.339959:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572630.339979:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572630.339981:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f900. 00000400:00000200:0.0:1788572630.339984:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572630.339987:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572630.339990:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1d48 00000400:00000010:0.0:1788572630.339992:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1d48. 00000100:00000001:0.0:1788572630.339994:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572630.339998:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed69180 x1875440809996800/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340018:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572630.340021:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440809996800/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340028:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.340029:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572630.340039:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572630.340041:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fb00. 00000400:00000200:0.0:1788572630.340043:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572630.340047:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572630.340049:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1770 00000400:00000010:0.0:1788572630.340051:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1770. 00000100:00000001:0.0:1788572630.340053:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572630.340057:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6b480 x1875440809996416/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340064:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572630.340067:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440809996416/t0(0) o400->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340075:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.340076:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572630.340084:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572630.340087:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f600. 00000400:00000200:0.0:1788572630.340090:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572630.340093:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572630.340096:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1c38 00000400:00000010:0.0:1788572630.340098:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1c38. 00000100:00000001:0.0:1788572630.340101:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572630.340104:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ed6ad80 x1875440809996672/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340113:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572630.340116:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440809996672/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572646 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572630.340133:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.340134:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572631.361739:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572631.361751:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572631.361783:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572631.361788:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572631.361790:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572631.361794:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572631.361798:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.257854:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572632.257859:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572632.257863:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572632.385851:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572632.385861:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572632.385864:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.385870:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572632.385874:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572632.385878:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.385881:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.409763:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572633.409779:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572633.409784:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572633.409795:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.409803:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572633.409813:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.409818:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.423424:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.423434:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572633.423441:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572633.423460:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440809996928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.423538:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572633.423545:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572633.423555:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572633.423564:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572633.423578:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572633.423608:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440809996928:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572633.423632:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572633.423681:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572633.423686:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572633.423708:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572633.423722:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572633.423733:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572633.423756:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572633.423766:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788572633.423792:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572633.423805:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809996928, portal 25 00000100:00000001:0.0:1788572633.423819:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572633.423829:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788572633.423851:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440809996928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.423926:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572633.423945:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572633.423955:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572633.423969:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440809996928, offset 0 00000400:00000200:0.0:1788572633.423995:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572633.424025:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572633.424047:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572633.424069:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572633.424085:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572633.424103:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572633.424116:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572633.424135:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540125:540125:256:4294967295] 192.168.202.148@tcp LPNI seq info [540090:540090:8:4294967295] 00000400:00000200:0.0:1788572633.424159:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572633.424192:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572633.424228:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572633.424244:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572633.424269:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572633.424296:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572633.424310:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1788572633.424343:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572633.424361:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572633.424369:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307776236 00000100:00000001:0.0:1788572633.424384:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4307773151 00000100:00000001:0.0:1788572633.424394:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307776236 00000100:00000001:0.0:1788572633.424403:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4307773151 00000100:00000001:0.0:1788572633.424410:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572633.424418:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424423:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424431:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572633.424437:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572633.424439:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572633.424445:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.424447:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572633.424449:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424452:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424468:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.424472:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572633.424513:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572633.424518:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0500. 00000400:00000200:0.0:1788572633.424527:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572633.424538:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572633.424547:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572633.424552:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572633.424556:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572633.424562:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5500 x1875440809996928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.424582:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572633.424590:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440809996928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572643 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.424612:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424617:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572633.424665:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572633.424669:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572633.424675:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572633.424684:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572633.424688:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572633.424705:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572633.424712:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572633.424721:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.424740:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572633.424749:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572718 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.424773:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.424781:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572633.424821:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788572633.424827:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572633.424829:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572633.424836:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572633.424838:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572633.424840:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.424842:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572633.424844:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572633.424853:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440809997184/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.424869:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572633.424874:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572633.424878:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572633.424880:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572633.424882:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572633.424887:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809997184:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572633.424904:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572633.424909:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572633.424912:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572633.424916:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572633.424923:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de6800. 02000000:00000001:0.0:1788572633.424929:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572633.424946:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572633.424956:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d68. 00000400:00000010:0.0:1788572633.424965:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572633.424973:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809997184, portal 4 00000100:00000001:0.0:1788572633.424983:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572633.424991:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788572633.425008:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440809997184/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.425029:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572633.425038:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572633.425044:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572633.425054:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1875440809997184, offset 0 00000400:00000200:0.0:1788572633.425071:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572633.425086:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572633.425098:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572633.425105:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572633.425115:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572633.425123:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572633.425132:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572633.425141:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540127:540127:256:4294967295] 192.168.202.148@tcp LPNI seq info [540092:540092:8:4294967295] 00000400:00000200:0.0:1788572633.425157:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572633.425178:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572633.425197:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572633.425203:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788572633.425215:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572633.425236:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572633.425241:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572633.425259:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572633.425267:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572633.425276:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572633.425285:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307772591 00000100:00000001:0.0:1788572633.425294:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307776236 00000100:00000001:0.0:1788572633.425304:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307772591 00000100:00000001:0.0:1788572633.425310:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307776236 00000100:00000001:0.0:1788572633.425318:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572633.425326:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425333:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425342:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425349:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572633.425353:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572633.425359:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425364:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572633.425388:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572633.425392:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0500. 00000400:00000200:0.0:1788572633.425399:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572633.425410:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572633.425413:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572633.425418:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572633.425422:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572633.425426:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c7100 x1875440809997184/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.425440:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572633.425447:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440809997184/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572649 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.425461:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425466:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572633.425507:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572633.425513:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572633.425518:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572633.425529:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572633.425535:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3088 00000400:00000010:0.0:1788572633.425539:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3088. 00000100:00000001:0.0:1788572633.425544:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572633.425553:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.425576:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572633.425582:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572649 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572633.425590:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572633.425594:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572633.425610:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13110) 00000100:00000001:0.0:1788572634.433756:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572634.433768:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572634.433772:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=10 : 10 : a) 00000100:00000001:0.0:1788572634.433778:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572634.433782:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572634.433786:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572634.433790:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572635.458787:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572635.458795:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f700. 00000400:00000200:0.0:1788572635.458804:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572635.458817:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572635.458828:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1f68 00000400:00000010:0.0:1788572635.458840:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1f68. 00000100:00000001:0.0:1788572635.458852:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572635.458862:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec70700 x1875440809997568/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572720 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.458894:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572635.458905:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70700 x1875440809997568/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572720 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.458977:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572635.458984:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572635.459121:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572635.459129:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498e00. 00000400:00000200:0.0:1788572635.459140:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572635.459155:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572635.459187:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c17f8 00000400:00000010:0.0:1788572635.459191:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c17f8. 00000100:00000001:0.0:1788572635.459199:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572635.459208:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71180 x1875440809997824/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459235:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572635.459242:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71180 x1875440809997824/t0(0) o400->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572651 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459256:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572635.459260:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572635.459272:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572635.459277:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788572635.459285:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572635.459296:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572635.459302:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572635.459307:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572635.459314:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572635.459318:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572655 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459333:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572635.459340:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572655 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459356:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572635.459362:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572635.459368:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572635.459377:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fe00. 00000400:00000200:0.0:1788572635.459382:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572635.459388:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572635.459391:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1e58 00000400:00000010:0.0:1788572635.459395:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1e58. 00000100:00000001:0.0:1788572635.459399:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572635.459403:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c371ec71880 x1875440809997952/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459408:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572635.459411:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71880 x1875440809997952/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 0 dl 1788572651 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572635.459416:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572635.459417:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788572636.482349:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572636.482359:0:232849:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572636.482366:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572636.482374:0:232849:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572636.482383:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572636.482391:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=9 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572636.482400:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572636.482405:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572636.482413:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572636.482425:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0500. 00000100:00000001:0.0:1788572636.482432:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.482439:0:232849:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572636.482451:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572636.482455:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.482461:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.482762:0:232849:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572636.482769:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572636.482775:0:232849:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572636.482781:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572636.482785:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572636.482788:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572636.482790:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572636.482795:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572636.482801:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 12707 00000400:00000200:0.0:1788572636.482810:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572636.482816:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00000040:0.0:1788572636.482821:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.482824:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572636.482827:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.482829:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572636.482832:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572636.482835:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572636.482837:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572636.482840:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572636.482842:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572636.482845:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572636.482848:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.482850:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572636.482851:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.482854:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572636.482857:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572636.482860:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572636.482871:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572636.482874:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788572636.482883:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572636.482885:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572636.482888:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572636.482890:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1788572636.482892:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572636.482896:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572636.482898:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572636.482900:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572636.482901:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572636.482903:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572636.482909:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.482916:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572636.482917:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572636.482921:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572636.482924:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572636.482927:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572636.482930:0:2403:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809998080:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572636.482933:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572636.482939:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572636.482942:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572636.482945:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572636.482950:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de6200. 02000000:00000001:0.0:1788572636.482952:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572636.482958:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572636.482962:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572636.482968:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572636.482974:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809998080, portal 10 00000100:00000001:0.0:1788572636.482977:0:2403:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572636.482979:0:2403:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000040:0.0:1788572636.482986:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.483007:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572636.483011:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572636.483014:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572636.483018:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809998080, offset 0 00000400:00000200:0.0:1788572636.483024:0:2403:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572636.483032:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572636.483039:0:2403:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572636.483046:0:2403:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572636.483050:0:2403:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572636.483054:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572636.483056:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572636.483060:0:2403:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540133:540133:256:4294967295] 192.168.202.148@tcp LPNI seq info [540098:540098:8:4294967295] 00000400:00000200:0.0:1788572636.483065:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572636.483075:0:2403:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572636.483082:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572636.483088:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788572636.483095:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572636.483106:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572636.483111:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572636.483123:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.483128:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572636.483131:0:2403:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572636.483135:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307775513 00000100:00000001:0.0:1788572636.483138:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307776236 00000100:00000001:0.0:1788572636.483140:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307775513 00000100:00000001:0.0:1788572636.483142:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307776236 00000100:00000001:0.0:1788572636.483145:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572636.483148:0:2403:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483157:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483159:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572636.483196:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572636.483200:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788572636.483205:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572636.483212:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572636.483217:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31550 00000400:00000010:0.0:1788572636.483219:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31550. 00000100:00000001:0.0:1788572636.483222:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572636.483227:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.483235:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572636.483238:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572721 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.483244:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483246:0:2396:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788572636.483265:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572636.483268:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483272:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572636.483276:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572636.483279:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483281:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572636.483285:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572636.483288:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572636.483292:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572636.483294:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483296:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572636.483298:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572636.483302:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572636.483308:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572636.483313:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572636.483319:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440809998208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572636.483339:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572636.483342:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.483347:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440809998208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.483383:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572636.483386:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572636.483390:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572636.483391:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572636.483393:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572636.483396:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572636.483399:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 12935 00000400:00000200:0.0:1788572636.483404:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572636.483408:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00000040:0.0:1788572636.483411:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.483413:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572636.483415:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.483418:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572636.483420:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572636.483421:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572636.483424:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572636.483426:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572636.483428:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572636.483431:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572636.483434:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.483436:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572636.483437:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572636.483439:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572636.483442:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572636.483452:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572636.483455:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572636.483457:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572636.483461:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572636.483463:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483465:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572636.483468:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572636.483471:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483472:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572636.483476:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572636.483479:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=7 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572636.483482:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572636.483483:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572636.483485:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572636.483486:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572636.483488:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572636.483492:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572636.483496:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572636.483501:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440809998336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572636.483510:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572636.483513:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.483518:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440809998336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572636.483526:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572636.483531:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 2 (13113) 00000100:00000040:0.0:1788572637.249750:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307776236, at 4307776256 00000100:00000010:0.0:1788572637.249768:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000020:00000001:0.0:1788572637.377924:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572637.377932:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572637.377936:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572637.505741:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572637.505765:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572637.505768:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=7 : 7 : 7) 00000100:00000001:0.0:1788572637.505773:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572637.505780:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572637.505783:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572637.505790:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572638.529760:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572638.529767:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572638.529771:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572638.529774:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572638.529785:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572638.529794:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=10 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572638.529801:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572638.529807:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572638.529822:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572638.529835:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0700. 00000100:00000001:0.0:1788572638.529845:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572638.529865:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b6d80 x1875440809998464/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572638.529893:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572638.529901:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572638.529912:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6d80 x1875440809998464/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.529964:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572638.529967:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572638.529970:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572638.529973:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572638.529976:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572638.529980:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572638.530015:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572638.530025:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 2 (13115) 00000100:00000001:0.0:1788572638.530054:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572638.530058:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572638.530063:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572638.530068:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572638.530555:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572638.530559:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572638.530567:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572638.530572:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572638.530578:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788572638.530580:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788572638.530584:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572638.530589:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b6d80 x1875440809998464/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.530599:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572638.530603:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440809998464/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572723 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.530610:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572638.530612:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572638.554697:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307777560, at 4307777561 00000100:00000010:0.0:1788572638.554711:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000080:00000001:0.0:1788572639.553752:0:189934:0:(rw.c:1480:ll_writepages()) Process entered 00000080:00000001:0.0:1788572639.553756:0:189934:0:(file.c:5295:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1788572639.553760:0:189934:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572639.553766:0:189934:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572639.553772:0:189934:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000020:00000001:0.0:1788572639.553780:0:189934:0:(cl_object.c:143:cl_object_top()) ffff9c3732214148 -> ffff9c3732214148 00000020:00000001:0.0:1788572639.553785:0:189934:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572639.553788:0:189934:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572639.553794:0:189934:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdff:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572639.553798:0:189934:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572639.553802:0:189934:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572639.553815:0:189934:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572639.553822:0:189934:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdff:0x0]io ffff9c371ff53a30 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1788572639.553827:0:189934:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3703d527c0 00020000:00000001:0.0:1788572639.553830:0:189934:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572639.553833:0:189934:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572639.553836:0:189934:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572639.553838:0:189934:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553840:0:189934:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572639.553842:0:189934:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572639.553847:0:189934:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553854:0:189934:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572639.553862:0:189934:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572639.553866:0:189934:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572639.553874:0:189934:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff9c3730ca38b8) owner 0000000000000000/ffff9c3703d527c0 00000020:00000001:0.0:1788572639.553884:0:189934:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553890:0:189934:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572639.553893:0:189934:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572639.553896:0:189934:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572639.553900:0:189934:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572639.553904:0:189934:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572639.553910:0:189934:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572639.553912:0:189934:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572639.553914:0:189934:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572639.553918:0:189934:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572639.553921:0:189934:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000020:00000001:0.0:1788572639.553925:0:189934:0:(cl_object.c:143:cl_object_top()) ffff9c3732214850 -> ffff9c3732214148 00000020:00000001:0.0:1788572639.553928:0:189934:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572639.553932:0:189934:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572639.553935:0:189934:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572639.553937:0:189934:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553940:0:189934:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553942:0:189934:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446634359293064416 : -109714416487200 : ffff9c371f2db4e0) 00000020:00000001:0.0:1788572639.553945:0:189934:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1788572639.553947:0:189934:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553950:0:189934:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553952:0:189934:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553959:0:189934:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572639.553961:0:189934:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572639.553963:0:189934:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572639.553973:0:189934:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572639.553975:0:189934:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572639.553976:0:189934:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553978:0:189934:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553980:0:189934:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.553982:0:189934:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553983:0:189934:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572639.553985:0:189934:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553986:0:189934:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.553988:0:189934:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1788572639.553990:0:189934:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572639.553992:0:189934:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572639.553993:0:189934:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572639.553995:0:189934:0:(osc_io.c:1099:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1788572639.553997:0:189934:0:(osc_cache.c:3326:osc_cache_writeback_range()) Process entered 00000008:00000020:0.0:1788572639.554002:0:189934:0:(osc_cache.c:3466:osc_cache_writeback_range()) obj ffff9c3713d8b500 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:0.0:1788572639.554007:0:189934:0:(osc_cache.c:3481:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572639.554009:0:189934:0:(osc_io.c:1168:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.554011:0:189934:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554013:0:189934:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554015:0:189934:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.554016:0:189934:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.554018:0:189934:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572639.554020:0:189934:0:(lov_io.c:1882:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1788572639.554022:0:189934:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572639.554023:0:189934:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1788572639.554024:0:189934:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572639.554025:0:189934:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554027:0:189934:0:(lov_io.c:1893:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1788572639.554029:0:189934:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572639.554030:0:189934:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572639.554032:0:189934:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572639.554033:0:189934:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572639.554034:0:189934:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572639.554036:0:189934:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572639.554037:0:189934:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554038:0:189934:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554040:0:189934:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572639.554041:0:189934:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572639.554043:0:189934:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572639.554044:0:189934:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572639.554045:0:189934:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572639.554053:0:189934:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572639.554055:0:189934:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572639.554056:0:189934:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572639.554057:0:189934:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572639.554059:0:189934:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554061:0:189934:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572639.554062:0:189934:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572639.554064:0:189934:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572639.554065:0:189934:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572639.554067:0:189934:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572639.554069:0:189934:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572639.554071:0:189934:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572639.554072:0:189934:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572639.554073:0:189934:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572639.554075:0:189934:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00020000:00000001:0.0:1788572639.554078:0:189934:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572639.554080:0:189934:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572639.554082:0:189934:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572639.554084:0:189934:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdff:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572639.554087:0:189934:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572639.554088:0:189934:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572639.554090:0:189934:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000080:00000001:0.0:1788572639.554093:0:189934:0:(file.c:5332:cl_sync_file_range()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572639.554095:0:189934:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572640.577797:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572640.577806:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572640.577809:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572640.577817:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572640.577833:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572640.577841:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=11 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572640.577852:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572640.577857:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572640.577870:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572640.577884:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0000. 00000100:00000001:0.0:1788572640.577901:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572640.577928:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b5880 x1875440809998592/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572640.577971:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572640.577977:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572640.577989:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5880 x1875440809998592/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572640.578022:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572640.578025:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572640.578030:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572640.578034:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572640.578038:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572640.578042:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572640.578048:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572640.578058:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 1 (13117) 00000100:00000001:0.0:1788572640.578085:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572640.578089:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572640.578098:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572640.578106:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572640.578571:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572640.578576:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572640.578584:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572640.578589:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572640.578595:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3d48 00000400:00000010:0.0:1788572640.578597:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3d48. 00000100:00000001:0.0:1788572640.578602:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572640.578607:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b5880 x1875440809998592/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572725 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572640.578618:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572640.578622:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5880 x1875440809998592/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572725 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572640.578659:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572640.578662:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572640.602701:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307779608, at 4307779609 00000100:00000010:0.0:1788572640.602716:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00080000:0.0:1788572641.601790:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572641.601802:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572641.601808:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572641.601815:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572641.601831:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572641.601843:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=12 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572641.601850:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572641.601862:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572641.601875:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572641.601895:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0900. 00000100:00000001:0.0:1788572641.601908:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572641.601921:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572641.601957:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572641.601966:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572641.601984:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572641.602060:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572641.602071:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572641.602094:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572641.602104:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572641.602121:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572641.602149:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572641.602163:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572641.602179:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13118) 00000100:00000001:0.0:1788572641.602230:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572641.602239:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572641.602251:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572641.602262:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572641.627721:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307780633, at 4307780634 00000100:00000010:0.0:1788572641.627752:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788572642.497865:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572642.497871:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572642.497875:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572643.650246:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572643.650252:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572643.650254:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572643.650257:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572643.650261:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572643.650265:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13105 00000400:00000200:0.0:1788572643.650272:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572643.650279:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00001000:0.0:1788572643.650283:0:189934:0:(import.c:2195:obd_at_measure()) add 10 to ffff9c373030dc18 time=58 v=5 (1 1 1 1) 00000100:00001000:0.0:1788572643.650287:0:189934:0:(import.c:2261:obd_at_measure()) AT ffff9c373030dc18 change: old=5 new=10 delta=5 (val=10) hist 10 1 1 1 00000100:00080000:0.0:1788572643.650290:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 5s 00000100:00000040:0.0:1788572643.650294:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572643.650298:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572643.650301:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572643.650303:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572643.650306:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572643.650309:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572643.650313:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572643.650326:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572643.650332:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572643.650335:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572643.650338:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572643.650341:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572643.650342:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572643.650344:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572643.650348:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572643.650351:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572643.650354:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572643.650358:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572643.650362:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572643.650365:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.650367:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572643.650370:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572643.650375:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572643.650392:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572643.650396:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572643.650397:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.650400:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572643.650403:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572643.650408:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572643.650417:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572643.650425:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572643.650434:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572643.650446:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572643.650451:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572643.650456:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.650896:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572643.650901:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572643.650915:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572643.650919:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572643.650922:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=13 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572643.650925:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572643.650928:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572643.650931:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572643.650936:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0e00. 00000100:00000001:0.0:1788572643.650940:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572643.650946:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572643.650955:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572643.650958:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572643.650964:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.650987:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572643.650990:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572643.650995:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572643.650998:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572643.651001:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572643.651005:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572643.651008:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572643.651015:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13120) 00000800:00000200:0.0:1788572643.651050:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572643.651054:0:2395:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9e00. 00000400:00000200:0.0:1788572643.651067:0:2395:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572643.651072:0:2395:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572643.651079:0:2395:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788572643.651087:0:2395:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788572643.651091:0:2395:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572643.651095:0:2395:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651103:0:2395:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572643.651106:0:2395:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572658 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651112:0:2395:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651114:0:2395:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572643.651132:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572643.651134:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572643.651136:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572643.651140:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651147:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572643.651149:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572643.651152:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572643.651154:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572643.651156:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572643.651159:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809998976:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572643.651162:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572643.651166:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572643.651168:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572643.651169:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572643.651173:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de7600. 02000000:00000001:0.0:1788572643.651176:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572643.651180:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572643.651183:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8318. 00000400:00000010:0.0:1788572643.651187:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788572643.651192:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809998976, portal 10 00000100:00000001:0.0:1788572643.651194:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572643.651196:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000040:0.0:1788572643.651203:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651218:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572643.651221:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572643.651223:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572643.651226:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809998976, offset 0 00000400:00000200:0.0:1788572643.651230:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572643.651235:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572643.651239:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572643.651243:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572643.651245:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572643.651248:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572643.651251:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572643.651254:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540140:540140:256:4294967295] 192.168.202.148@tcp LPNI seq info [540105:540105:8:4294967295] 00000400:00000200:0.0:1788572643.651258:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572643.651265:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572643.651270:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572643.651274:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572643.651278:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572643.651284:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572643.651287:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572643.651298:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572643.651302:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572643.651306:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572643.651308:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307782681 00000100:00000001:0.0:1788572643.651311:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307786463 00000100:00000001:0.0:1788572643.651314:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307782681 00000100:00000001:0.0:1788572643.651316:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307786463 00000100:00000001:0.0:1788572643.651318:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572643.651321:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651331:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651333:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572643.651339:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572643.651342:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572643.651343:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572643.651344:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572643.651346:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651349:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651371:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572643.651373:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572643.651376:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651397:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572643.651420:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572643.651423:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572643.651427:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572643.651431:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572643.651435:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572643.651437:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572643.651441:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572643.651446:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651456:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572643.651459:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440809998976/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572728 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572643.651465:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572643.651467:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572644.609702:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572644.609715:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572644.609718:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572644.609724:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572644.609728:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572644.609732:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572644.609735:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572644.609897:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572644.609922:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572644.609927:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572644.609929:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572644.673757:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572644.673766:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572644.673770:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572644.673776:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572644.673781:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572644.673793:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572644.673803:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572645.633732:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572645.633745:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572645.633748:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572645.633754:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572645.633756:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572645.633760:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572645.633764:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572645.697735:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572645.697749:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572645.697753:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572645.697761:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572645.697771:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572645.697776:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572645.697780:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.657756:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572646.657777:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572646.657780:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.657787:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.657790:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.657796:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.657800:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.721771:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572646.721788:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572646.721792:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.721802:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.721807:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.721820:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.721833:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724023:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.724030:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.724035:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572646.724046:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440809999232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572646.724125:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572646.724135:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572646.724153:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572646.724162:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572646.724175:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572646.724189:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440809999232:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572646.724210:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572646.724223:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572646.724236:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572646.724249:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572646.724266:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572646.724272:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572646.724302:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572646.724309:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8e70. 00000400:00000010:0.0:1788572646.724321:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572646.724334:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809999232, portal 4 00000100:00000001:0.0:1788572646.724340:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572646.724345:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788572646.724356:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440809999232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572646.724378:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572646.724386:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572646.724392:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572646.724402:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440809999232, offset 0 00000400:00000200:0.0:1788572646.724421:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572646.724438:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572646.724461:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572646.724472:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572646.724482:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572646.724492:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572646.724503:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572646.724520:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540142:540142:255:4294967295] 192.168.202.148@tcp LPNI seq info [540107:540107:7:4294967295] 00000400:00000200:0.0:1788572646.724539:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572646.724571:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572646.724589:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572646.724597:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572646.724606:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572646.724612:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572646.724616:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0f00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1788572646.724631:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572646.724662:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572646.724666:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572646.724669:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307785891 00000100:00000001:0.0:1788572646.724675:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307786463 00000100:00000001:0.0:1788572646.724679:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307785891 00000100:00000001:0.0:1788572646.724684:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307786463 00000100:00000001:0.0:1788572646.724687:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572646.724693:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724702:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724708:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.724718:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572646.724723:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.724730:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.724735:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.724740:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572646.724752:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440809999360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572646.724784:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572646.724788:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572646.724795:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572646.724800:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572646.724811:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572646.724818:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440809999360:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572646.724821:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572646.724823:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572646.724824:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572646.724825:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572646.724828:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572646.724830:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572646.724832:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572646.724835:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8478. 00000400:00000010:0.0:1788572646.724838:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572646.724840:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809999360, portal 4 00000100:00000001:0.0:1788572646.724842:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572646.724843:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788572646.724848:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440809999360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572646.724854:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572646.724856:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572646.724857:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572646.724859:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440809999360, offset 0 00000400:00000200:0.0:1788572646.724862:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572646.724865:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572646.724867:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572646.724870:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572646.724872:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572646.724873:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572646.724875:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572646.724878:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540143:540143:256:4294967295] 192.168.202.148@tcp LPNI seq info [540108:540108:8:4294967295] 00000400:00000200:0.0:1788572646.724881:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572646.724886:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572646.724889:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572646.724896:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572646.724899:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572646.724903:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572646.724905:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0200 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1788572646.724916:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572646.724919:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572646.724921:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572646.724923:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307785900 00000100:00000001:0.0:1788572646.724925:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307786463 00000100:00000001:0.0:1788572646.724926:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307785900 00000100:00000001:0.0:1788572646.724928:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307786463 00000100:00000001:0.0:1788572646.724930:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572646.724932:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724935:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724937:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.724940:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572646.724941:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.724943:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.724944:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.724945:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.724947:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572647.489786:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307786463, at 4307786496 00000100:00000010:0.0:1788572647.489813:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788572647.617820:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572647.617825:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572647.617830:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572647.681767:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572647.681787:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572647.681790:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572647.681797:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572647.681800:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572647.681805:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572647.681810:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572647.745809:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572647.745829:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572647.745836:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572647.745844:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572647.745851:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572647.745862:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572647.745870:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572648.705771:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572648.705788:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572648.705790:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572648.705796:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572648.705799:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572648.705803:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572648.705808:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572648.769850:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572648.769859:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572648.769865:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572648.769871:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572648.769882:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572648.769891:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=15 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572648.769897:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572648.769901:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572648.769910:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572648.769920:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0b00. 00000100:00000001:0.0:1788572648.769929:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572648.769945:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572648.769960:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572648.769965:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572648.769972:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572648.770016:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572648.770019:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572648.770036:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572648.770040:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572648.770044:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572648.770048:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572648.770051:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572648.770057:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 3 (13125) 00000100:00000001:0.0:1788572648.770073:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572648.770075:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572648.770078:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572648.770081:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1788572648.770083:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572648.770086:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572648.770088:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572648.770090:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572648.770091:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572648.770093:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572648.770099:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572648.770108:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572648.770112:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572648.770117:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572648.770121:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572648.770124:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572648.770128:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809999488:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572648.770133:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572648.770147:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572648.770151:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572648.770154:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572648.770159:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de7200. 02000000:00000001:0.0:1788572648.770161:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572648.770178:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572648.770183:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8160. 00000400:00000010:0.0:1788572648.770192:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788572648.770202:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809999488, portal 10 00000100:00000001:0.0:1788572648.770205:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572648.770207:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000040:0.0:1788572648.770217:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572648.770241:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572648.770244:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572648.770248:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572648.770252:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809999488, offset 0 00000400:00000200:0.0:1788572648.770260:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572648.770270:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572648.770279:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572648.770288:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572648.770294:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572648.770299:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572648.770302:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572648.770307:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540144:540144:256:4294967295] 192.168.202.148@tcp LPNI seq info [540109:540109:8:4294967295] 00000400:00000200:0.0:1788572648.770313:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572648.770325:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572648.770332:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572648.770338:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572648.770347:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (5) 00000800:00000200:0.0:1788572648.770356:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.148@tcp ip 192.168.202.148:988 00000800:00000200:0.0:1788572648.770362:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c36010e0100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572648.770379:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572648.770385:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572648.770392:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572648.770400:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307787800 00000100:00000010:0.0:1788572648.770417:0:2404:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572648.770432:0:2404:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572648.770444:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307787800 00000100:00000010:0.0:1788572648.770450:0:2404:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626480. 00000100:00000001:0.0:1788572648.770455:0:2404:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572648.770467:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572648.770472:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572648.770478:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572648.770482:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572648.770521:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788572648.770526:0:2396:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0100. 00000400:00000200:0.0:1788572648.770532:0:2396:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788572648.770538:0:2396:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: OK 00000400:00000200:0.0:1788572648.770547:0:2396:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31ee0 00000400:00000010:0.0:1788572648.770550:0:2396:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31ee0. 00000100:00000001:0.0:1788572648.770555:0:2396:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572648.770561:0:2396:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572648.770572:0:2396:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572648.770576:0:2396:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440809999488/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572648.770585:0:2396:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572648.770586:0:2396:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572648.794762:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307787800, at 4307787801 00000100:00000010:0.0:1788572648.794782:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572649.793704:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572649.793713:0:2404:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572649.793728:0:2404:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572633/real 1788572633] req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572649.793751:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572649.793768:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572649.793772:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f29f8. 00000400:00000200:0.0:1788572649.793778:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788572649.793782:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788572649.793790:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572649.793798:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572649.793821:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572649.793829:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572649.793832:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572649.793835:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572649.793837:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572649.793842:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572649.793850:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572649.793853:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9c372073d000 already not connected (conn 9, was 7): CONNECTING 00000100:00000001:0.0:1788572649.793857:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572649.793858:0:2404:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572649.793863:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572649.793865:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=64 : 64 : 40) 00000100:00000001:0.0:1788572649.793868:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572649.793870:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572649.793873:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572649.793876:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572649.793878:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572649.793883:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572649.793892:0:2404:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572649.793896:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572649.793897:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572649.793899:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572649.793903:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572649.793918:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809997312:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572649.793923:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572649.793926:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572649.793930:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440809997312/t0(0) o400->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572649 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572649.793937:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572649.793940:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572649.793944:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3711215400. 02000000:00000001:0.0:1788572649.793950:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572649.793953:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572649.793955:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572649.793957:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572649.793960:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9900. 02000000:00000001:0.0:1788572649.793965:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572649.793966:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572649.793968:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788572649.793972:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572649.793974:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572649.793978:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572651.777861:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(24) -108 00000100:00000001:0.0:1788572651.777925:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572651.777931:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572651.777937:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572651.777950:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440809999616/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572651.777965:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572651.777970:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572651.777979:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572651.777982:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572651.777986:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572651.777992:0:2404:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809999616:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572651.778008:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572651.778015:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572651.778018:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572651.778021:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572651.778027:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9c3704de7000. 02000000:00000001:0.0:1788572651.778029:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572651.778036:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572651.778040:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8370. 00000400:00000010:0.0:1788572651.778047:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572651.778054:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440809999616, portal 10 00000100:00000001:0.0:1788572651.778057:0:2404:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572651.778059:0:2404:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788572651.778067:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440809999616/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572651.778077:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572651.778081:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572651.778085:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572651.778089:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440809999616, offset 0 00000400:00000200:0.0:1788572651.778097:0:2404:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572651.778108:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572651.778119:0:2404:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572651.778149:0:2404:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572651.778158:0:2404:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572651.778163:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572651.778167:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572651.778171:0:2404:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540145:540145:256:4294967295] 192.168.202.148@tcp LPNI seq info [540110:540110:8:4294967295] 00000400:00000200:0.0:1788572651.778177:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572651.778186:0:2404:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572651.778193:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572651.778198:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572651.778204:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572651.778236:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572651.778241:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572651.778253:0:2404:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572651.778258:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307790808 00000100:00000010:0.0:1788572651.778263:0:2404:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572651.778271:0:2404:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572651.778277:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307790808 00000100:00000010:0.0:1788572651.778281:0:2404:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626000. 00000100:00000001:0.0:1788572651.778285:0:2404:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572651.778295:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572651.778299:0:2404:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572651.778304:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572651.778307:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572651.778315:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572651.778320:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=62 : 62 : 3e) 00000100:00000001:0.0:1788572651.778323:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572651.778325:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572651.778329:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572651.778332:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572651.802695:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307790808, at 4307790809 00000100:00000010:0.0:1788572651.802708:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572651.841765:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572651.841776:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572651.841780:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572651.841784:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572651.841790:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572651.841795:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572651.841802:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572652.737866:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572652.737870:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572652.737873:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572652.865716:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572652.865735:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572652.865738:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572652.865764:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572652.865767:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572652.865771:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572652.865774:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572653.889763:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572653.889781:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572653.889784:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572653.889791:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572653.889793:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572653.889799:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572653.889802:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572654.913712:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572654.913723:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572654.913727:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572654.913732:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572654.913735:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572654.913739:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572654.913744:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.937725:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572655.937732:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572655.937748:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572635/real 1788572635] req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572655.937760:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572655.937762:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572655.937765:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2528. 00000400:00000200:0.0:1788572655.937769:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788572655.937772:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788572655.937775:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572655.937777:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572655.937783:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.937788:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572655.937789:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937791:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572655.937792:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572655.937810:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.937815:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937822:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572655.937823:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937826:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572655.937828:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572655.937831:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572655.937836:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.937848:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572655.937851:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937853:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572655.937854:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937856:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572655.937858:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572655.937861:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572655.937886:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572655.937890:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572655.937895:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572655.937903:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440809997696:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572655.937908:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.937910:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572655.937913:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73b80 x1875440809997696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572655 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.937921:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572655.937924:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572655.937928:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788572655.937946:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572655.937950:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572655.937951:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572655.937954:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572655.937956:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788572655.937961:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572655.937964:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572655.937969:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73b80. 00000100:00000001:0.0:1788572655.937974:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572655.937976:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.937981:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.938889:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572655.938893:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572655.938895:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572655.938900:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.938909:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572655.938911:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572655.938915:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572655.938917:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572655.938920:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572655.938923:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440809999872:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572655.938926:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572655.938931:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572655.938933:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572655.938949:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572655.938955:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572655.938957:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572655.938964:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572655.938968:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572655.938975:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572655.938981:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440809999872, portal 10 00000100:00000001:0.0:1788572655.938984:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572655.938986:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788572655.938994:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572655.939020:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572655.939024:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572655.939027:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788572655.939031:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440809999872, offset 0 00000400:00000200:0.0:1788572655.939038:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572655.939046:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572655.939052:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572655.939058:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572655.939062:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572655.939067:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572655.939069:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572655.939073:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540146:540146:255:4294967295] 192.168.202.148@tcp LPNI seq info [540111:540111:7:4294967295] 00000400:00000200:0.0:1788572655.939078:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572655.939089:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572655.939096:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572655.939101:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572655.939108:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572655.939118:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572655.939123:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572655.939127:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572655.939129:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307794973 00000100:00000010:0.0:1788572655.939133:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572655.939140:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572655.939146:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307794973 00000100:00000010:0.0:1788572655.939150:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626000. 00000100:00000001:0.0:1788572655.939154:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572655.939174:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572655.939177:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.939181:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.939191:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.939197:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572655.939199:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572655.939201:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572655.939202:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572655.939203:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572655.939206:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572655.967704:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307794973, at 4307794974 00000100:00000010:0.0:1788572655.967720:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572656.961851:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572656.961862:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572656.961864:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572656.961867:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572656.961869:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572656.961874:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572656.961876:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572657.857870:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572657.857875:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572657.857879:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572657.985760:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572657.985780:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572657.985783:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572657.985790:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572657.985794:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572657.985801:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572657.985805:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.009684:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572659.009691:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572659.009704:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572643/real 1788572643] req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572659.009721:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572659.009723:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572659.009726:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2bb0. 00000400:00000200:0.0:1788572659.009729:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3330 00000400:00000010:0.0:1788572659.009731:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3330. 00000100:00000001:0.0:1788572659.009734:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572659.009737:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572659.009755:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.009761:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572659.009763:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.009766:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572659.009768:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572659.009772:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.009778:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.009785:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572659.009787:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.009790:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572659.009792:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572659.009796:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572659.009800:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.009807:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572659.009809:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.009811:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572659.009812:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.009814:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572659.009816:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572659.009818:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572659.009833:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572659.009847:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572659.009852:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572659.009860:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440809998848:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572659.009883:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.009885:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572659.009889:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440809998848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572658 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.009896:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572659.010040:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572659.010044:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788572659.010049:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572659.010051:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572659.010054:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572659.010056:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572659.010059:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572659.010063:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572659.010065:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572659.010067:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788572659.010070:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572659.010072:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.010074:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.010161:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572659.010162:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572659.010164:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572659.010168:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.010176:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572659.010179:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572659.010182:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572659.010183:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572659.010186:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572659.010189:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810000128:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572659.010193:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572659.010197:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572659.010199:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572659.010201:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572659.010205:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572659.010212:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572659.010229:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572659.010232:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d68. 00000400:00000010:0.0:1788572659.010238:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572659.010245:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810000128, portal 25 00000100:00000001:0.0:1788572659.010250:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572659.010253:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788572659.010261:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572679 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572659.010283:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572659.010289:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572659.010293:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31220. 00000100:00000200:0.0:1788572659.010299:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810000128, offset 0 00000400:00000200:0.0:1788572659.010310:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572659.010319:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572659.010327:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572659.010335:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572659.010342:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572659.010348:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572659.010353:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572659.010361:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540148:540148:253:4294967295] 192.168.202.148@tcp LPNI seq info [540113:540113:5:4294967295] 00000400:00000200:0.0:1788572659.010369:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572659.010382:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572659.010394:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572659.010402:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572659.010414:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572659.010436:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572659.010443:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572659.010448:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307801822 00000100:00000010:0.0:1788572659.010455:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572659.010463:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572659.010476:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307801822 00000100:00000010:0.0:1788572659.010483:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626480. 00000100:00000001:0.0:1788572659.010489:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572659.010496:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572659.010502:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.010507:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.010510:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.010516:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572659.010520:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.010525:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572659.010528:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572659.010533:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.010538:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572660.033787:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572660.033808:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572660.033811:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572660.033819:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572660.033821:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572660.033827:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572660.033830:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572661.058842:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572661.058857:0:234251:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572661.058860:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572661.058863:0:234251:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572661.058866:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572661.058869:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572661.058871:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572661.058874:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572661.058877:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572661.058881:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13118 00000400:00000200:0.0:1788572661.058885:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572661.058895:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00001000:0.0:1788572661.058899:0:234251:0:(import.c:2195:obd_at_measure()) add 15 to ffff9c3707ccac18 time=126 v=10 (10 1 2 0) 00000100:00001000:0.0:1788572661.058902:0:234251:0:(import.c:2261:obd_at_measure()) AT ffff9c3707ccac18 change: old=10 new=15 delta=5 (val=15) hist 15 1 2 0 00000100:00080000:0.0:1788572661.058905:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 10s 00000100:00000040:0.0:1788572661.058908:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.058911:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572661.058913:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.058915:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572661.058918:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572661.058919:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572661.058922:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572661.058925:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572661.058927:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572661.058929:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572661.058932:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.058934:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572661.058936:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.058937:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572661.058940:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572661.058943:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572661.058946:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572661.058948:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572661.058953:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572661.058956:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.058958:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572661.058962:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572661.058964:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.058966:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572661.058970:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572661.058973:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572661.058975:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572661.058982:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.058984:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572661.058987:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572661.058989:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572661.058994:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572661.058997:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572661.059003:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572661.059011:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572661.059013:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572661.059017:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572661.059038:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572661.059041:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572661.059043:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572661.059045:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572661.059046:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572661.059048:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572661.059051:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13118 00000400:00000200:0.0:1788572661.059056:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572661.059060:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00001000:0.0:1788572661.059062:0:234251:0:(import.c:2195:obd_at_measure()) add 15 to ffff9c372073d418 time=48 v=10 (10 1 0 0) 00000100:00001000:0.0:1788572661.059066:0:234251:0:(import.c:2261:obd_at_measure()) AT ffff9c372073d418 change: old=10 new=15 delta=5 (val=15) hist 15 1 0 0 00000100:00080000:0.0:1788572661.059072:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 10s 00000100:00000040:0.0:1788572661.059079:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.059084:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572661.059088:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.059092:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572661.059097:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572661.059099:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572661.059108:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572661.059117:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572661.059121:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572661.059125:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572661.059129:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.059133:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572661.059138:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572661.059141:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572661.059143:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572661.059147:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572661.059150:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572661.059154:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572661.059158:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572661.059162:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.059166:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572661.059170:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572661.059174:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.059178:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572661.059182:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572661.059187:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572661.059191:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572661.059193:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.059196:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572661.059199:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572661.059201:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572661.059221:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572661.059228:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572661.059236:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572661.059252:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572661.059258:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572661.059267:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572661.059297:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572661.059301:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 3 (13137) 00000100:00000040:0.0:1788572662.849782:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307801822, at 4307801856 00000100:00000010:0.0:1788572662.849810:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788572662.977823:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572662.977826:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572662.977829:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572664.065702:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572664.065710:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572664.065713:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572664.065716:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572664.065723:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572664.065728:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=20 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572664.065731:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572664.065734:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572664.065738:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572664.065744:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0d00. 00000100:00000001:0.0:1788572664.065750:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572664.065770:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b6300 x1875440810000768/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572664.065778:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572664.065781:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572664.065786:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b6300 x1875440810000768/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572664.065808:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572664.065809:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572664.065812:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572664.065824:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572664.065834:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572664.065837:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572664.065841:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572664.065846:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 2 (13140) 00000100:00000001:0.0:1788572664.065893:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572664.065896:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572664.065902:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572664.065906:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572664.090724:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307803096, at 4307803097 00000100:00000010:0.0:1788572664.090739:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788572666.113748:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572666.113760:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572666.113764:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572666.113769:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572666.113780:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572666.113785:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=21 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572666.113790:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572666.113793:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572666.113798:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572666.113808:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0600. 00000100:00000001:0.0:1788572666.113815:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572666.113826:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b5c00 x1875440810000896/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572666.113838:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572666.113842:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572666.113849:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b5c00 x1875440810000896/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572666.113901:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572666.113904:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572666.113908:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572666.113911:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572666.113914:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572666.113917:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572666.113921:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572666.113926:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13142) 00000100:00000001:0.0:1788572666.113976:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572666.113982:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572666.113989:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.114000:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572666.138777:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307805144, at 4307805145 00000100:00000010:0.0:1788572666.138794:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788572668.097823:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572668.097828:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572668.097831:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572670.273835:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572670.273850:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572670.273853:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572670.273858:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572670.273861:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572670.273866:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572670.273869:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572671.297794:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572671.297811:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572671.297815:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572671.297830:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572671.297841:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572671.297849:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572671.297871:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572671.322704:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307810328, at 4307810329 00000100:00000010:0.0:1788572671.322721:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215d20. 00000100:00000001:0.0:1788572672.321704:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572672.321723:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572672.321726:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572672.321734:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572672.321737:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572672.321743:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572672.321747:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572673.217806:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572673.217810:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572673.217813:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572673.345721:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572673.345743:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572673.345746:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572673.345751:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572673.345754:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572673.345760:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572673.345763:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572674.369679:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572674.369695:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572674.369697:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572674.369702:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572674.369705:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572674.369709:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572674.369711:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572675.393709:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572675.393721:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572675.393723:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572675.393727:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572675.393729:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572675.393734:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572675.393736:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572676.417713:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572676.417724:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572676.417739:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572676.417746:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572676.417750:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572676.417766:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572676.417772:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572676.442704:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307815448, at 4307815449 00000100:00000010:0.0:1788572676.442720:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215180. 00000100:00000001:0.0:1788572677.441693:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572677.441704:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572677.441706:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572677.441710:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572677.441713:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572677.441716:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572677.441719:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572678.337833:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572678.337838:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572678.337842:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572678.465694:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572678.465710:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572678.465712:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572678.465718:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572678.465720:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572678.465725:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572678.465728:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572679.489833:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572679.489841:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572679.489858:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1788572659/real 0] req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572679.489878:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572679.489880:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572679.489884:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d68. 00000400:00000200:0.0:1788572679.489889:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572679.489892:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572679.489897:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572679.489903:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572679.489915:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572679.489946:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788572679.489952:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to UnregRPC req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572679.489962:0:2402:0:(client.c:3018:ptlrpc_unregister_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572679.489965:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572679.489968:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572679.489973:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 2 fl UnregRPC:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572679.489981:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572679.489990:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572679.489993:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572679.489995:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572679.489998:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572679.490003:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572679.490006:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572680.513833:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572680.513844:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572680.513857:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1788572655/real 0] req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572680.513872:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572680.513874:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572680.513877:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572680.513881:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572680.513883:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572680.513886:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572680.513889:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572680.513897:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572680.513906:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788572680.513926:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to UnregRPC req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572680.513934:0:2402:0:(client.c:3018:ptlrpc_unregister_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572680.513937:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572680.513940:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572680.513944:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 2 fl UnregRPC:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572680.513953:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572680.513961:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572680.513963:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572680.513966:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572680.513969:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572680.513973:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572680.513976:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.537852:0:2402:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572681.537857:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572681.537883:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1788572661/real 0] req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572681.537915:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572681.537919:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572681.537924:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2058. 00000400:00000200:0.0:1788572681.537931:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788572681.537935:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788572681.537942:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572681.537948:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572681.537960:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.537979:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788572681.537987:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to UnregRPC req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.538020:0:2402:0:(client.c:3018:ptlrpc_unregister_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538026:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572681.538030:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572681.538040:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl UnregRPC:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.538062:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572681.538068:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572681.538076:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1788572661/real 0] req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572681.538097:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572681.538099:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572681.538101:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2e18. 00000400:00000200:0.0:1788572681.538107:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3330 00000400:00000010:0.0:1788572681.538111:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3330. 00000100:00000001:0.0:1788572681.538115:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572681.538121:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572681.538136:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.538148:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788572681.538156:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to UnregRPC req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl Rpc:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.538168:0:2402:0:(client.c:3018:ptlrpc_unregister_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538171:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572681.538174:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572681.538188:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 2 fl UnregRPC:XNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572681.538217:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572681.538270:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572681.538274:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538279:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572681.538283:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572681.538290:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538296:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538328:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572681.538332:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572681.538347:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572681.538353:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572681.562715:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307820568, at 4307820569 00000100:00000010:0.0:1788572681.562728:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000800:00000200:0.0:1788572682.497940:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788572683.457774:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572683.457778:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572683.457781:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572686.657782:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572686.657790:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572686.657793:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572686.657796:0:189934:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572686.657803:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572686.657808:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=25 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000100:00000001:0.0:1788572686.657812:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572686.657815:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572686.657819:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572686.657827:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9c36010e0200. 00000100:00000001:0.0:1788572686.657834:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.657844:0:189934:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID req@ffff9c372d6b6a00 x1875440810001536/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572686.657853:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572686.657857:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.657863:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_00_01+1] req@ffff9c372d6b6a00 x1875440810001536/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.657924:0:189934:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572686.657930:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572686.657934:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572686.657940:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572686.657949:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572686.657955:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572686.657962:0:189934:0:(pinger.c:247:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572686.657971:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13163) 00000100:00000001:0.0:1788572686.659350:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572686.659352:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572686.659354:0:2403:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572686.659359:0:2403:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572681/real 1788572686] req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.659367:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.659368:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.659369:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2108. 00000400:00000200:0.0:1788572686.659372:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f32a8 00000400:00000010:0.0:1788572686.659373:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f32a8. 00000100:00000001:0.0:1788572686.659375:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.659377:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.659383:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.659387:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.659388:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.659389:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.659395:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.659399:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.659404:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572686.675401:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9c3707cce000 already not connected (conn 3, was 3): DISCONN 00000100:00000001:0.0:1788572686.675406:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572686.675408:0:2403:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.675420:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675427:0:2403:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000001:0.0:1788572686.675437:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675442:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.675446:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.675457:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810001408/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572766 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.675491:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440810001408:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572686.675521:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572686.675527:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788572686.675554:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from UnregRPC to Rpc req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675592:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.675605:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675619:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.675625:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675629:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.675633:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675660:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.675672:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.675677:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.675693:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.675699:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.675708:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.675734:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810000640:192.168.202.148@tcp:8:kworker.0 00000100:00000040:0.0:1788572686.675750:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from UnregRPC to Rpc req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675762:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.675769:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675780:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.675786:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675790:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.675794:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675798:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.675803:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.675808:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.675820:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.675827:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.675836:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.675866:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810000512:192.168.202.148@tcp:8:kworker.0 00000100:00000040:0.0:1788572686.675882:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from UnregRPC to Rpc req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 1 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675913:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.675923:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.675956:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.675964:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675970:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.675973:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.675977:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.675981:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.675988:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.676000:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.676012:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.676023:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.676051:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810000128:192.168.202.148@tcp:250:kworker.0 00000100:00000040:0.0:1788572686.676068:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from UnregRPC to Rpc req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 1 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676101:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.676109:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676123:0:2402:0:(client.c:2150:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.676129:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676135:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.676138:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676144:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.676148:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.676160:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.676181:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.676188:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.676201:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.676221:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440809999872:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572686.676233:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676240:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.676251:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7480 x1875440810000640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676283:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.676287:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.676292:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c373233f400. 02000000:00000001:0.0:1788572686.676298:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.676302:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.676307:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572686.676314:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.676319:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788572686.676325:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.676329:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.676333:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7480. 00000100:00000001:0.0:1788572686.676340:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.676343:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676349:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.676356:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810000512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572681 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676372:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.676375:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.676379:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788572686.676386:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.676389:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.676398:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572686.676403:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.676406:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572686.676411:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.676414:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.676417:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788572686.676421:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.676424:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676427:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.676435:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810000128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572679 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676448:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.676452:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.676457:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572686.676463:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.676467:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.676472:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572686.676478:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.676481:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788572686.676489:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.676493:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.676499:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788572686.676506:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.676510:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676515:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.676530:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440809999872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676566:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.676572:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.676578:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572686.676585:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.676589:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.676592:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572686.676599:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.676602:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788572686.676608:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.676617:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.676621:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788572686.676628:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.676632:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676655:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.676662:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572686.676666:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.676673:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572686.676677:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572686.676682:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572686.676690:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676733:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572686.676736:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.676741:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572686.676746:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572686.676754:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572686.676763:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572686.676773:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572686.676779:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572686.676783:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.676789:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572686.676796:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572686.676802:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.676818:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572686.676824:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572686.676833:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572686.676841:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810001664, portal 10 00000100:00000001:0.0:1788572686.676845:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572686.676849:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337920 : -109714423213696 : ffff9c371ec71180) 00000100:00000040:0.0:1788572686.676863:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572696 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.676898:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572686.676906:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572686.676917:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572686.676931:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810001664, offset 0 00000400:00000200:0.0:1788572686.676944:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572686.676956:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.676967:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572686.676974:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572686.676983:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572686.676991:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572686.676998:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572686.677008:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540159:540159:256:4294967295] 192.168.202.148@tcp LPNI seq info [540124:540124:8:4294967295] 00000400:00000200:0.0:1788572686.677021:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.677041:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572686.677059:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.677068:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572686.677082:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572686.677104:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572686.677136:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.677140:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572686.677148:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572686.677161:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572686.677172:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572686.677179:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572686.677187:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572686.677204:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572696 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677248:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.677258:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572696 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677292:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.677297:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572686.677313:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.677319:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572686.677330:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572686.677337:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307825707 00000100:00000001:0.0:1788572686.677349:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307825688 00000100:00000001:0.0:1788572686.677354:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307825707 00000100:00000001:0.0:1788572686.677362:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307825688 00000100:00000001:0.0:1788572686.677368:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572686.677375:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.677381:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572686.677389:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572686/real 1788572686] req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.677423:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.677426:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.677431:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572686.677437:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572686.677441:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572686.677447:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.677457:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.677500:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677530:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.677537:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.677546:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.677552:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.677563:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677585:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.677601:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677620:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.677626:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.677631:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.677634:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.677655:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.677656:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.677659:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.677663:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.677668:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.677681:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.677718:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572686.677725:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572686.677733:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677743:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572686.677746:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.677750:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572686.677753:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572686.677757:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572686.677762:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001792:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572686.677765:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572686.677767:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572686.677768:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.677770:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572686.677776:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572686.677780:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.677795:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572686.677798:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572686.677802:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572686.677807:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810001792, portal 4 00000100:00000001:0.0:1788572686.677812:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572686.677815:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286344192 : -109714423207424 : ffff9c371ec72a00) 00000100:00000040:0.0:1788572686.677825:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677831:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572686.677834:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572686.677836:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572686.677838:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810001792, offset 0 00000400:00000200:0.0:1788572686.677841:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572686.677845:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.677849:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572686.677854:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572686.677858:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572686.677861:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572686.677865:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572686.677869:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540160:540160:256:4294967295] 192.168.202.148@tcp LPNI seq info [540125:540125:8:4294967295] 00000400:00000200:0.0:1788572686.677880:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.677889:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572686.677895:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.677898:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572686.677904:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572686.677908:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572686.677925:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.677927:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572686.677931:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572686.677938:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572686.677942:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572686.677949:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572686.677953:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572686.677959:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677976:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.677980:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.677989:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.677991:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572686.677994:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.677999:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572686.678004:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572686.678006:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307825844 00000100:00000001:0.0:1788572686.678012:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307825707 00000100:00000001:0.0:1788572686.678018:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307825844 00000100:00000001:0.0:1788572686.678024:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307825707 00000100:00000001:0.0:1788572686.678030:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572686.678034:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678039:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572686.678049:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572686/real 1788572686] req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.678064:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.678068:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.678071:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572686.678077:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572686.678083:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572686.678101:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.678109:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.678131:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678153:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.678158:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678162:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678166:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.678177:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678192:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.678202:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678216:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.678219:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678223:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678226:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678230:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.678234:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.678240:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.678248:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.678254:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.678264:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.678276:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001792:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572686.678282:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678285:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.678290:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71180 x1875440810001664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572696 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678299:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.678302:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.678311:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572686.678317:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.678320:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.678322:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=15 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572686.678325:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.678327:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2dc00. 02000000:00000001:0.0:1788572686.678333:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.678335:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.678338:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71180. 00000100:00000001:0.0:1788572686.678340:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.678343:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678345:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.678351:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810001792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678361:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.678364:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.678366:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572686.678368:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.678369:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.678370:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572686.678372:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.678373:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788572686.678374:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.678375:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.678376:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788572686.678378:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.678379:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678380:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678383:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572686.678384:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678385:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572686.678387:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572686.678388:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572686.678390:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678394:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572686.678398:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678399:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572686.678401:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572686.678402:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572686.678405:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572686.678407:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572686.678408:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572686.678410:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678411:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572686.678413:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572686.678414:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.678417:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572686.678419:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572686.678421:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572686.678423:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810001920, portal 4 00000100:00000001:0.0:1788572686.678425:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572686.678426:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286341504 : -109714423210112 : ffff9c371ec71f80) 00000100:00000040:0.0:1788572686.678430:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678434:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572686.678436:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572686.678437:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572686.678439:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810001920, offset 0 00000400:00000200:0.0:1788572686.678442:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678444:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678447:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572686.678449:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572686.678451:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572686.678453:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572686.678455:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678457:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540161:540161:256:4294967295] 192.168.202.148@tcp LPNI seq info [540126:540126:8:4294967295] 00000400:00000200:0.0:1788572686.678459:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678464:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572686.678471:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678473:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572686.678475:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572686.678477:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572686.678486:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678488:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572686.678491:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572686.678494:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572686.678498:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572686.678500:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572686.678503:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572686.678506:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678514:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.678518:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678525:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678526:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572686.678530:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.678533:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572686.678537:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572686.678539:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307825853 00000100:00000001:0.0:1788572686.678544:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, simply extend its deadline from 4307825844 00000100:00000001:0.0:1788572686.678548:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307825853 00000100:00000001:0.0:1788572686.678551:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, simply extend its deadline from 4307825844 00000100:00000001:0.0:1788572686.678554:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572686.678556:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678558:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572686.678563:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572686/real 1788572686] req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.678578:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.678580:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.678582:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572686.678585:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572686.678586:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572686.678589:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.678592:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.678598:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678603:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.678604:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678605:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678606:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.678608:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678613:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.678615:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678620:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.678621:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678622:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678623:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678624:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.678625:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.678627:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.678629:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.678630:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.678659:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.678666:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810001920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572686.678670:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572686.678672:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678677:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572686.678678:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678680:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572686.678682:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572686.678684:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572686.678686:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810002048:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572686.678689:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572686.678691:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572686.678693:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678694:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572686.678697:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572686.678698:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.678702:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572686.678704:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572686.678706:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572686.678709:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810002048, portal 25 00000100:00000001:0.0:1788572686.678710:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572686.678711:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788572686.678714:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678719:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572686.678720:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572686.678722:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572686.678723:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810002048, offset 0 00000400:00000200:0.0:1788572686.678731:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678734:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678736:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572686.678739:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572686.678741:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572686.678742:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572686.678744:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678746:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540162:540162:256:4294967295] 192.168.202.148@tcp LPNI seq info [540127:540127:8:4294967295] 00000400:00000200:0.0:1788572686.678749:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678753:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572686.678756:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678758:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572686.678761:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572686.678763:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572686.678770:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678772:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572686.678774:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572686.678776:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572686.678778:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572686.678780:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572686.678781:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572686.678784:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678789:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.678791:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678795:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678796:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572686.678798:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.678800:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572686.678802:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307829491 00000100:00000001:0.0:1788572686.678808:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4307825853 00000100:00000001:0.0:1788572686.678810:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307829491 00000100:00000001:0.0:1788572686.678812:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4307825853 00000100:00000001:0.0:1788572686.678814:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572686.678815:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.678816:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572686.678819:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572686/real 1788572686] req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.678824:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.678825:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.678827:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572686.678828:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572686.678829:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572686.678831:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.678832:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.678837:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678842:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.678843:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678844:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678845:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.678847:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678851:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.678853:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678857:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.678858:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678862:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.678863:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.678865:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.678866:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.678868:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.678870:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.678877:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.678881:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.678885:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810002048:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572686.678888:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572686.678890:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678895:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572686.678896:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678898:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572686.678899:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572686.678901:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572686.678904:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810002176:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572686.678906:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572686.678908:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572686.678909:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572686.678911:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572686.678913:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572686.678914:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572686.678916:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572686.678918:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572686.678920:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572686.678922:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810002176, portal 10 00000100:00000001:0.0:1788572686.678924:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572686.678928:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788572686.678932:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.678936:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572686.678938:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572686.678939:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572686.678941:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810002176, offset 0 00000400:00000200:0.0:1788572686.678943:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678946:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678948:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572686.678950:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572686.678952:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572686.678954:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572686.678955:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572686.678957:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540163:540163:256:4294967295] 192.168.202.148@tcp LPNI seq info [540128:540128:8:4294967295] 00000400:00000200:0.0:1788572686.678960:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572686.678964:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572686.678967:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678969:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572686.678971:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572686.678973:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572686.678980:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572686.678982:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0f00. 00000400:00000200:0.0:1788572686.678984:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572686.678986:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572686.678987:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31440 00000400:00000010:0.0:1788572686.678989:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31440. 00000100:00000001:0.0:1788572686.678990:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572686.678993:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679005:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.679007:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572711 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679011:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.679012:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572686.679015:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.679017:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572686.679019:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572686.679020:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307825713 00000100:00000001:0.0:1788572686.679022:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307829491 00000100:00000001:0.0:1788572686.679024:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307825713 00000100:00000001:0.0:1788572686.679025:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307829491 00000100:00000001:0.0:1788572686.679027:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572686.679028:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.679029:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572686.679031:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572686/real 1788572686] req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572686.679036:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572686.679037:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572686.679038:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572686.679039:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572686.679040:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572686.679042:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572686.679044:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572686.679049:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679052:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572686.679056:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679058:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.679058:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572686.679061:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679066:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572686.679068:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679073:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572686.679074:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679075:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572686.679076:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679077:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572686.679078:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572686.679079:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572686.679081:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572686.679082:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572686.679085:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572686.679102:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810002176:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572686.679104:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679106:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.679108:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810001920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679112:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.679114:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.679115:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788572686.679117:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.679118:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.679123:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572686.679124:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.679125:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788572686.679127:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.679128:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.679129:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788572686.679131:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.679132:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679133:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.679136:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810002048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679140:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.679141:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.679142:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788572686.679144:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.679145:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.679145:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572686.679147:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.679147:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788572686.679149:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.679149:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.679150:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788572686.679151:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.679152:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679153:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572686.679155:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810002176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572711 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572686.679158:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572686.679159:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572686.679160:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788572686.679161:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572686.679162:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572686.679162:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572686.679163:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572686.679164:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788572686.679165:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572686.679169:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572686.679171:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788572686.679172:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572686.679173:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679174:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572686.679177:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572686.679178:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572686.679179:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572686.679180:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572688.577787:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572688.577791:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572688.577793:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572690.497770:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307829491, at 4307829504 00000100:00000010:0.0:1788572690.497792:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00080000:0.0:1788572691.777878:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572691.777890:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572691.777893:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572691.777897:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572691.777901:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572691.777909:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13158 00000400:00000200:0.0:1788572691.777919:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572691.777926:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572691.777929:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572691.777933:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.777937:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.777940:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.777943:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572691.777947:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572691.777950:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572691.777952:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572691.777957:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572691.777981:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572691.777984:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572691.778002:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778004:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778007:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778008:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572691.778011:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572691.778015:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778018:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572691.778021:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572691.778025:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572691.778028:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778031:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572691.778033:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572691.778039:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572691.778043:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572691.778047:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572691.778048:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778052:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572691.778054:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572691.778058:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572691.778066:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572691.778073:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572691.778083:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810002304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572691.778095:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572691.778099:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.778104:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810002304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572691.778132:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572691.778135:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572691.778139:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572691.778140:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572691.778148:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572691.778152:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572691.778156:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13158 00000400:00000200:0.0:1788572691.778161:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572691.778166:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00001000:0.0:1788572691.778170:0:189934:0:(import.c:2195:obd_at_measure()) add 10 to ffff9c3707cce418 time=137 v=5 (1 1 1 1) 00000100:00001000:0.0:1788572691.778174:0:189934:0:(import.c:2261:obd_at_measure()) AT ffff9c3707cce418 change: old=5 new=10 delta=5 (val=10) hist 10 1 1 1 00000100:00080000:0.0:1788572691.778177:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 5s 00000100:00000040:0.0:1788572691.778180:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778182:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778185:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778188:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572691.778194:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572691.778197:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572691.778200:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572691.778206:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572691.778213:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572691.778216:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572691.778227:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778234:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778239:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778242:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572691.778244:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572691.778246:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778248:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572691.778255:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572691.778264:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572691.778269:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778273:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572691.778277:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572691.778292:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778298:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572691.778307:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572691.778315:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=16 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572691.778320:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572691.778324:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778331:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572691.778339:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572691.778346:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572691.778352:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572691.778364:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572691.778376:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810002432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572691.778396:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572691.778404:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.778414:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810002432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572691.778428:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572691.778432:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572691.778440:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572691.778442:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572691.778444:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572691.778448:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572691.778456:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13158 00000400:00000200:0.0:1788572691.778468:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572691.778479:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572691.778492:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572691.778510:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778521:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778530:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778544:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572691.778552:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572691.778556:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572691.778558:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572691.778561:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572691.778568:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572691.778577:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572691.778583:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778584:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778586:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778588:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572691.778591:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572691.778595:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778600:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572691.778607:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572691.778612:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572691.778619:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778623:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572691.778631:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572691.778689:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778692:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572691.778699:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572691.778705:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572691.778708:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572691.778714:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.778721:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572691.778723:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572691.778727:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572691.778736:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572691.778744:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572691.778757:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810002560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572691.778794:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572691.778796:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.778802:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810002560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572691.778810:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572691.778813:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572691.778819:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572691.778825:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572691.778828:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572691.778834:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572691.778841:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13158 00000400:00000200:0.0:1788572691.778850:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572691.778859:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572691.778866:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572691.778873:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778879:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778882:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778886:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572691.778894:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572691.778900:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572691.778905:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572691.778910:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572691.778914:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572691.778927:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572691.778940:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778949:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.778972:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.778976:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572691.778997:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572691.779009:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779019:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572691.779025:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572691.779033:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572691.779044:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779053:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572691.779060:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572691.779068:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779074:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572691.779084:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572691.779095:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572691.779102:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572691.779107:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779112:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572691.779118:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572691.779121:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572691.779127:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572691.779133:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572691.779146:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810002688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572691.779179:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572691.779187:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.779198:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810002688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572691.779219:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572691.779229:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572691.779246:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572691.779252:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572691.779256:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572691.779269:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572691.779286:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13158 00000400:00000200:0.0:1788572691.779301:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572691.779314:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572691.779318:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572691.779328:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779335:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.779348:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779357:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572691.779370:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572691.779381:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572691.779387:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572691.779394:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572691.779405:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572691.779413:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572691.779430:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779441:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572691.779453:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779464:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572691.779472:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572691.779485:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572691.779491:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572691.779500:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572691.779509:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572691.779514:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779516:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572691.779522:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572691.779532:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779539:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572691.779548:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572691.779556:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572691.779566:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572691.779578:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572691.779584:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572691.779590:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572691.779595:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572691.779606:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572691.779618:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572691.779631:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810002816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572691.779685:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572691.779690:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.779702:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810002816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572691.779727:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572691.779738:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13168) 00000020:00000001:0.0:1788572693.697810:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572693.697815:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572693.697820:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572695.617754:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307834591, at 4307834624 00000100:00000010:0.0:1788572695.617769:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215d20. 00000100:00000001:0.0:1788572696.899066:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572696.899073:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572696.899077:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572696.899087:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899097:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572696.899099:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899104:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572696.899108:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572696.899111:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572696.899116:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810002944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572696.899122:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572696.899129:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572696.899142:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899146:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572696.899152:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572696.899154:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.899163:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572696.899168:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572696.899178:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572696.899189:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810002944, portal 25 00000100:00000001:0.0:1788572696.899192:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572696.899194:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345088 : -109714423206528 : ffff9c371ec72d80) 00000100:00000040:0.0:1788572696.899201:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899211:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572696.899216:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572696.899220:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31440. 00000100:00000200:0.0:1788572696.899223:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810002944, offset 0 00000400:00000200:0.0:1788572696.899232:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899244:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899253:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572696.899260:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572696.899266:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572696.899272:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572696.899276:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899281:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540169:540169:256:4294967295] 192.168.202.148@tcp LPNI seq info [540134:540134:8:4294967295] 00000400:00000200:0.0:1788572696.899287:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899298:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572696.899308:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572696.899316:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788572696.899326:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572696.899358:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.899363:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572696.899367:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307839711 00000100:00000010:0.0:1788572696.899394:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572696.899404:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572696.899414:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307839711 00000100:00000010:0.0:1788572696.899418:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626660. 00000100:00000001:0.0:1788572696.899422:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572696.899426:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572696.899429:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.899433:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572696.899438:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899446:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572696.899447:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899450:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572696.899452:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572696.899454:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572696.899457:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572696.899460:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572696.899463:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572696.899465:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899466:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572696.899470:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572696.899473:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.899477:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572696.899481:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d68. 00000400:00000010:0.0:1788572696.899487:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572696.899490:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003072, portal 10 00000100:00000001:0.0:1788572696.899493:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572696.899494:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788572696.899503:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899509:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572696.899517:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572696.899520:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31220. 00000100:00000200:0.0:1788572696.899523:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810003072, offset 0 00000400:00000200:0.0:1788572696.899528:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899532:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899536:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572696.899539:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572696.899542:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572696.899544:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572696.899546:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899549:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540170:540170:255:4294967295] 192.168.202.148@tcp LPNI seq info [540135:540135:7:4294967295] 00000400:00000200:0.0:1788572696.899553:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899559:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572696.899564:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572696.899570:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572696.899574:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572696.899582:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572696.899587:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572696.899590:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572696.899592:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307835929 00000100:00000001:0.0:1788572696.899595:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899597:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307835929 00000100:00000001:0.0:1788572696.899600:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899602:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572696.899604:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.899607:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572696.899612:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899619:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572696.899621:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899630:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572696.899632:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572696.899661:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572696.899665:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572696.899669:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572696.899671:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572696.899673:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899674:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572696.899678:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572696.899680:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.899683:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572696.899686:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788572696.899690:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788572696.899693:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003200, portal 10 00000100:00000001:0.0:1788572696.899696:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572696.899697:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788572696.899703:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899709:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572696.899711:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572696.899713:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed312a8. 00000100:00000200:0.0:1788572696.899716:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810003200, offset 0 00000400:00000200:0.0:1788572696.899719:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899723:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899727:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572696.899729:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572696.899732:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572696.899734:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572696.899736:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899739:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540171:540171:254:4294967295] 192.168.202.148@tcp LPNI seq info [540136:540136:6:4294967295] 00000400:00000200:0.0:1788572696.899743:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899749:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572696.899759:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572696.899763:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572696.899768:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572696.899774:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572696.899778:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572696.899781:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572696.899783:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307835934 00000100:00000001:0.0:1788572696.899785:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899787:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307835934 00000100:00000001:0.0:1788572696.899790:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899792:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572696.899795:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.899797:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572696.899802:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899810:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572696.899811:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899813:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572696.899815:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572696.899817:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572696.899820:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572696.899823:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572696.899825:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572696.899827:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.899829:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572696.899867:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572696.899870:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.899874:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572696.899877:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788572696.899882:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572696.899892:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003328, portal 4 00000100:00000001:0.0:1788572696.899894:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572696.899895:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788572696.899903:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.899910:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572696.899913:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572696.899916:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31a18. 00000100:00000200:0.0:1788572696.899918:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810003328, offset 0 00000400:00000200:0.0:1788572696.899923:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899928:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899932:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572696.899934:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572696.899937:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572696.899939:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572696.899941:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572696.899945:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540172:540172:253:4294967295] 192.168.202.148@tcp LPNI seq info [540137:540137:5:4294967295] 00000400:00000200:0.0:1788572696.899949:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.899956:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572696.899961:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572696.899964:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572696.899968:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572696.899975:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572696.899979:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572696.899983:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572696.899984:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307836066 00000100:00000001:0.0:1788572696.899987:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899990:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307836066 00000100:00000001:0.0:1788572696.899992:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.899994:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572696.900002:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.900004:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572696.900009:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.900016:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572696.900018:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.900020:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572696.900022:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572696.900024:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572696.900028:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572696.900031:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572696.900033:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572696.900035:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572696.900037:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572696.900040:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788572696.900042:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572696.900046:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572696.900049:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572696.900053:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572696.900056:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003456, portal 4 00000100:00000001:0.0:1788572696.900059:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572696.900060:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788572696.900067:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572696.900074:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572696.900077:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572696.900079:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed313b8. 00000100:00000200:0.0:1788572696.900082:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810003456, offset 0 00000400:00000200:0.0:1788572696.900087:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572696.900091:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.900095:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572696.900098:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572696.900108:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572696.900110:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572696.900113:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572696.900117:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540173:540173:252:4294967295] 192.168.202.148@tcp LPNI seq info [540138:540138:4:4294967295] 00000400:00000200:0.0:1788572696.900122:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572696.900130:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572696.900136:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572696.900140:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572696.900146:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572696.900155:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572696.900159:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572696.900163:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572696.900165:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307836075 00000100:00000001:0.0:1788572696.900167:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.900169:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307836075 00000100:00000001:0.0:1788572696.900171:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307839711 00000100:00000001:0.0:1788572696.900173:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572696.900175:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.900177:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.900181:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572696.900188:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572696.900191:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572696.900194:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572696.900195:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572696.900197:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572696.900199:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.923130:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.923136:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.923140:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572697.923151:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572696/real 1788572697] req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572697.923183:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572697.923185:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572697.923189:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572697.923193:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572697.923195:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572697.923199:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572697.923204:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572697.923219:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923225:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572697.923227:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923230:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923233:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572697.923236:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923244:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572697.923249:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923257:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572697.923260:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923262:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923263:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923266:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572697.923268:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572697.923271:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572697.923276:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572697.923288:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572697.923294:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572697.923311:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810002944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572697.923317:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572697.923321:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572696/real 1788572697] req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572697.923330:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572697.923331:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572697.923334:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d68. 00000400:00000200:0.0:1788572697.923337:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31ee0 00000400:00000010:0.0:1788572697.923338:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31ee0. 00000100:00000001:0.0:1788572697.923341:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572697.923345:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572697.923353:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923359:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572697.923361:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923363:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923364:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572697.923367:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923387:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572697.923391:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923397:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572697.923400:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923401:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923402:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923410:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572697.923412:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572697.923415:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572697.923427:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572697.923430:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572697.923435:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572697.923442:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572697.923447:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572697.923451:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572696/real 1788572697] req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572697.923460:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572697.923462:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572697.923464:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8f78. 00000400:00000200:0.0:1788572697.923467:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788572697.923469:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788572697.923472:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572697.923476:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572697.923484:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923490:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572697.923491:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923493:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923494:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572697.923497:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923503:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572697.923514:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923522:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572697.923524:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923526:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923527:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923529:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572697.923531:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572697.923533:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572697.923537:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572697.923540:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572697.923544:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572697.923552:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572697.923557:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572697.923561:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572696/real 1788572697] req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572697.923570:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572697.923572:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572697.923574:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8210. 00000400:00000200:0.0:1788572697.923577:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572697.923579:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572697.923582:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572697.923585:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572697.923594:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923602:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572697.923611:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923613:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923615:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572697.923620:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923658:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572697.923664:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923671:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572697.923673:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923674:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923676:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923678:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572697.923679:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572697.923682:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572697.923685:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572697.923688:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572697.923692:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572697.923700:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572697.923705:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572697.923709:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572696/real 1788572697] req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572697.923719:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572697.923721:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572697.923723:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572697.923727:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572697.923730:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572697.923738:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572697.923743:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572697.923752:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923757:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572697.923758:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923760:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923761:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572697.923763:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923768:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572697.923770:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923775:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572697.923777:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923778:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572697.923780:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923781:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572697.923783:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572697.923786:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572697.923790:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572697.923793:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572697.923796:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572697.923817:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572697.923823:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923826:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572697.923838:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72d80 x1875440810002944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923846:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572697.923849:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572697.923853:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572697.923858:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572697.923861:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572697.923864:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572697.923866:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572697.923870:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788572697.923874:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572697.923876:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572697.923878:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72d80. 00000100:00000001:0.0:1788572697.923882:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572697.923884:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923886:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572697.923890:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810003072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923897:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572697.923899:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572697.923900:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572697.923903:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572697.923904:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572697.923906:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=15 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572697.923908:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572697.923909:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788572697.923911:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572697.923911:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572697.923912:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788572697.923914:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572697.923915:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923916:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572697.923918:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810003200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923922:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572697.923930:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572697.923932:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572697.923935:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572697.923936:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572697.923938:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572697.923940:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572697.923941:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788572697.923945:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572697.923946:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572697.923948:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788572697.923950:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572697.923952:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923954:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572697.923958:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810003328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923965:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572697.923966:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572697.923968:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572697.923970:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572697.923972:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572697.923973:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572697.923975:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572697.923976:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f400. 02000000:00000001:0.0:1788572697.923978:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572697.923979:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572697.923981:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788572697.923984:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572697.923985:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.923987:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572697.923990:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810003456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.923996:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572697.923997:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572697.923999:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788572697.924002:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572697.924003:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572697.924010:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572697.924011:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572697.924013:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788572697.924015:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572697.924017:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572697.924018:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788572697.924020:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572697.924022:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.924025:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.924033:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.924034:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.924037:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.924039:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572698.817823:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572698.817830:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572698.817836:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572699.586385:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572699.586390:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572699.586393:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572699.586402:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586419:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572699.586422:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.586427:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572699.586430:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572699.586434:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572699.586439:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810003712:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572699.586442:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572699.586445:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572699.586449:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.586453:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572699.586460:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572699.586464:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.586473:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572699.586477:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572699.586486:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572699.586499:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003712, portal 25 00000100:00000001:0.0:1788572699.586503:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572699.586506:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788572699.586512:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586522:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572699.586524:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572699.586528:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572699.586534:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810003712, offset 0 00000400:00000200:0.0:1788572699.586539:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572699.586546:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.586554:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572699.586559:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572699.586564:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572699.586568:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572699.586572:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572699.586579:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540174:540174:256:4294967295] 192.168.202.148@tcp LPNI seq info [540139:540139:8:4294967295] 00000400:00000200:0.0:1788572699.586587:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.586603:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572699.586614:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.586622:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572699.586631:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572699.586666:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1788572699.586703:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.586709:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572699.586717:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572699.586723:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572699.586732:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572699.586735:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572699.586741:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572699.586747:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586783:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.586793:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586809:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.586812:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572699.586820:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.586828:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572699.586832:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307842399 00000100:00000001:0.0:1788572699.586841:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4307839711 00000100:00000001:0.0:1788572699.586847:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307842399 00000100:00000001:0.0:1788572699.586853:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4307839711 00000100:00000001:0.0:1788572699.586858:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572699.586863:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.586867:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572699.586872:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572699/real 1788572699] req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572699.586890:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572699.586893:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572699.586897:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572699.586902:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572699.586906:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572699.586912:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572699.586918:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572699.586938:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586950:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572699.586953:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.586956:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.586966:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572699.586974:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.586990:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572699.586997:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587003:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572699.587008:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587012:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.587014:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587021:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572699.587024:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572699.587027:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572699.587032:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572699.587044:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572699.587057:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572699.587084:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810003712:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572699.587101:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587107:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.587117:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810003712/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587137:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572699.587141:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572699.587146:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572699.587150:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572699.587153:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572699.587156:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572699.587161:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572699.587171:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233cc00. 02000000:00000001:0.0:1788572699.587177:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572699.587180:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572699.587184:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788572699.587188:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572699.587191:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587196:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587204:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572699.587208:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587212:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572699.587215:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572699.587219:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572699.587226:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587242:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572699.587245:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587248:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572699.587252:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572699.587255:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572699.587261:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572699.587269:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572699.587272:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572699.587275:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587277:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572699.587280:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572699.587283:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.587291:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572699.587297:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572699.587303:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572699.587309:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003840, portal 10 00000100:00000001:0.0:1788572699.587314:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572699.587316:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788572699.587325:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587344:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572699.587346:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572699.587348:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572699.587351:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810003840, offset 0 00000400:00000200:0.0:1788572699.587354:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587357:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587360:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572699.587362:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572699.587364:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572699.587366:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572699.587368:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587370:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540175:540175:256:4294967295] 192.168.202.148@tcp LPNI seq info [540140:540140:8:4294967295] 00000400:00000200:0.0:1788572699.587373:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587377:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572699.587381:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587383:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572699.587386:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572699.587388:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1788572699.587396:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587397:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572699.587399:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572699.587402:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572699.587404:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572699.587405:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572699.587407:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572699.587410:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587415:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.587417:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587430:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587432:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572699.587434:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572699.587437:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572699.587439:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572699.587441:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307838617 00000100:00000001:0.0:1788572699.587443:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587445:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307838617 00000100:00000001:0.0:1788572699.587446:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587447:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572699.587448:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587450:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572699.587452:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572699/real 1788572699] req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572699.587457:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572699.587458:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572699.587459:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572699.587460:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572699.587461:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572699.587463:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572699.587465:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572699.587470:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587474:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572699.587475:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587476:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.587477:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572699.587479:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587486:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572699.587490:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587497:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572699.587500:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587501:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.587503:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587505:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572699.587506:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572699.587509:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572699.587513:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572699.587516:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572699.587520:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572699.587528:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572699.587532:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572699.587536:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587543:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572699.587544:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587547:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572699.587549:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572699.587551:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572699.587554:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003968:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572699.587557:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572699.587559:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572699.587567:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587569:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572699.587572:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572699.587574:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.587577:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572699.587580:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572699.587583:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572699.587586:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810003968, portal 10 00000100:00000001:0.0:1788572699.587588:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572699.587590:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291926656 : -109714417624960 : ffff9c371f1c5880) 00000100:00000040:0.0:1788572699.587596:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587603:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572699.587606:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572699.587609:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572699.587612:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810003968, offset 0 00000400:00000200:0.0:1788572699.587616:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587621:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587625:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572699.587628:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572699.587630:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572699.587632:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572699.587656:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587660:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540176:540176:256:4294967295] 192.168.202.148@tcp LPNI seq info [540141:540141:8:4294967295] 00000400:00000200:0.0:1788572699.587665:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587672:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572699.587677:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587680:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572699.587684:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572699.587686:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1788572699.587693:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587699:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572699.587701:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572699.587704:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572699.587707:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572699.587709:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572699.587712:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572699.587715:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587721:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.587723:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587727:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587729:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572699.587731:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572699.587733:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572699.587736:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572699.587737:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307838622 00000100:00000001:0.0:1788572699.587739:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587740:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307838622 00000100:00000001:0.0:1788572699.587742:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587743:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572699.587744:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587746:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572699.587748:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572699/real 1788572699] req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572699.587753:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572699.587754:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572699.587755:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572699.587757:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572699.587758:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572699.587763:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572699.587784:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572699.587790:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587794:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572699.587795:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587796:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.587797:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572699.587799:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587804:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572699.587807:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587811:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572699.587812:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587813:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.587814:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.587815:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572699.587816:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572699.587818:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572699.587820:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572699.587821:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572699.587823:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572699.587828:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810003968:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572699.587831:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572699.587837:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587841:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572699.587842:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587844:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572699.587845:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572699.587847:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572699.587848:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572699.587851:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572699.587852:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572699.587853:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.587854:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572699.587856:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572699.587858:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.587860:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572699.587862:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572699.587865:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572699.587867:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004096, portal 4 00000100:00000001:0.0:1788572699.587868:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572699.587869:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788572699.587873:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587877:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572699.587879:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572699.587880:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572699.587882:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810004096, offset 0 00000400:00000200:0.0:1788572699.587884:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587887:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587889:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572699.587891:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572699.587893:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572699.587895:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572699.587896:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572699.587902:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540177:540177:256:4294967295] 192.168.202.148@tcp LPNI seq info [540142:540142:8:4294967295] 00000400:00000200:0.0:1788572699.587905:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.587909:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572699.587912:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587915:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572699.587917:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572699.587919:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1788572699.587926:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.587928:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572699.587930:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572699.587932:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572699.587934:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572699.587935:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572699.587937:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572699.587939:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587945:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.587947:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587951:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587952:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572699.587954:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572699.587956:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572699.587958:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572699.587960:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307838754 00000100:00000001:0.0:1788572699.587962:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587963:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307838754 00000100:00000001:0.0:1788572699.587964:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.587966:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572699.587971:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.587972:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572699.587975:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572699/real 1788572699] req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572699.587981:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572699.587982:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572699.587983:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572699.587984:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572699.587985:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572699.587987:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572699.587988:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572699.587994:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.587998:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572699.587999:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588000:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.588001:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572699.588003:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588007:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572699.588009:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588013:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572699.588015:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588016:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.588016:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588017:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572699.588018:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572699.588020:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572699.588025:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572699.588026:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572699.588029:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572699.588034:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572699.588036:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588037:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.588039:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810003840/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588044:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572699.588045:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572699.588046:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572699.588048:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572699.588049:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572699.588050:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=15 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572699.588051:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572699.588053:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e800. 02000000:00000001:0.0:1788572699.588054:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572699.588056:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572699.588057:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788572699.588058:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572699.588059:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588060:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.588062:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440810003968/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588066:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572699.588067:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572699.588068:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572699.588069:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572699.588070:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572699.588071:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572699.588072:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572699.588075:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788572699.588077:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572699.588078:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572699.588079:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788572699.588080:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572699.588081:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588082:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.588084:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810004096/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588088:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572699.588089:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572699.588090:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572699.588091:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572699.588091:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572699.588092:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572699.588093:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572699.588094:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788572699.588095:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572699.588096:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572699.588097:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788572699.588098:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572699.588099:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588100:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588102:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572699.588103:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.588104:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572699.588105:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572699.588106:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572699.588108:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588112:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572699.588113:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.588114:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572699.588115:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572699.588116:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572699.588118:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004224:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572699.588123:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572699.588124:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572699.588126:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572699.588127:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572699.588129:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572699.588130:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572699.588132:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572699.588134:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572699.588136:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572699.588138:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004224, portal 4 00000100:00000001:0.0:1788572699.588139:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572699.588140:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291922176 : -109714417629440 : ffff9c371f1c4700) 00000100:00000040:0.0:1788572699.588144:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588148:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572699.588150:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572699.588151:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572699.588153:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810004224, offset 0 00000400:00000200:0.0:1788572699.588155:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572699.588157:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.588160:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572699.588162:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572699.588163:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572699.588165:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572699.588167:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572699.588169:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540178:540178:256:4294967295] 192.168.202.148@tcp LPNI seq info [540143:540143:8:4294967295] 00000400:00000200:0.0:1788572699.588171:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572699.588175:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572699.588178:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.588180:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572699.588182:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572699.588187:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1788572699.588197:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572699.588199:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0400. 00000400:00000200:0.0:1788572699.588201:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572699.588203:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572699.588206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31000 00000400:00000010:0.0:1788572699.588207:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31000. 00000100:00000001:0.0:1788572699.588208:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572699.588211:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588216:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.588217:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588222:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.588223:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572699.588224:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572699.588226:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572699.588228:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572699.588229:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307838763 00000100:00000001:0.0:1788572699.588231:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.588232:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307838763 00000100:00000001:0.0:1788572699.588234:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307842399 00000100:00000001:0.0:1788572699.588235:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572699.588236:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.588238:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572699.588240:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572699/real 1788572699] req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572699.588244:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572699.588245:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572699.588249:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572699.588251:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572699.588252:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572699.588253:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572699.588256:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572699.588261:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588265:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572699.588266:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588267:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.588268:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572699.588270:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588274:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572699.588277:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588280:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572699.588282:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588283:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572699.588283:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588285:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572699.588286:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572699.588287:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572699.588289:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572699.588290:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572699.588293:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572699.588298:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004224:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572699.588304:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588306:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572699.588308:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4700 x1875440810004224/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572699.588312:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572699.588313:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572699.588314:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572699.588316:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572699.588316:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572699.588317:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572699.588319:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572699.588320:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f400. 02000000:00000001:0.0:1788572699.588321:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572699.588322:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572699.588323:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4700. 00000100:00000001:0.0:1788572699.588325:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572699.588326:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588327:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.588329:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572699.588330:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.588331:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572699.588332:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572703.425715:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307842399, at 4307842432 00000100:00000010:0.0:1788572703.425730:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788572703.937889:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572703.937894:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572703.937898:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572705.090019:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.090026:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.090030:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572705.090042:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090058:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.090064:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090081:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572705.090085:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572705.090092:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.090100:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810004352:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572705.090108:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572705.090114:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572705.090118:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090123:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572705.090131:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572705.090138:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.090149:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572705.090156:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572705.090166:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572705.090177:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004352, portal 25 00000100:00000001:0.0:1788572705.090183:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572705.090188:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788572705.090200:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090218:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572705.090226:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572705.090232:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31000. 00000100:00000200:0.0:1788572705.090238:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810004352, offset 0 00000400:00000200:0.0:1788572705.090248:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572705.090261:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.090271:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572705.090278:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572705.090286:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572705.090296:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572705.090303:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572705.090312:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540179:540179:256:4294967295] 192.168.202.148@tcp LPNI seq info [540144:540144:8:4294967295] 00000400:00000200:0.0:1788572705.090323:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.090339:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572705.090358:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572705.090367:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788572705.090380:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572705.090413:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.090421:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572705.090425:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307847902 00000100:00000010:0.0:1788572705.090433:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572705.090443:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572705.090452:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307847902 00000100:00000010:0.0:1788572705.090459:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626540. 00000100:00000001:0.0:1788572705.090468:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572705.090492:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572705.090499:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.090506:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.090512:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.090524:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.090528:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572705.090535:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.090544:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.090548:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572705.090561:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090573:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.090577:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090582:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572705.090586:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572705.090592:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.090607:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572705.090616:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572705.090621:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572705.090623:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090672:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572705.090689:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572705.090696:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.090706:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572705.090712:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788572705.090720:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788572705.090726:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004480, portal 10 00000100:00000001:0.0:1788572705.090730:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572705.090733:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788572705.090748:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090769:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572705.090776:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572705.090781:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31ee0. 00000100:00000200:0.0:1788572705.090787:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810004480, offset 0 00000400:00000200:0.0:1788572705.090799:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572705.090806:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.090817:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572705.090823:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572705.090825:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572705.090827:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572705.090830:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572705.090833:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540180:540180:255:4294967295] 192.168.202.148@tcp LPNI seq info [540145:540145:7:4294967295] 00000400:00000200:0.0:1788572705.090838:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.090845:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572705.090850:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572705.090855:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788572705.090859:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572705.090868:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572705.090871:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572705.090875:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572705.090877:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307844121 00000100:00000001:0.0:1788572705.090886:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.090889:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307844121 00000100:00000001:0.0:1788572705.090890:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.090893:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572705.090895:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.090898:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572705.090903:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090910:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.090912:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090915:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572705.090916:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572705.090919:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.090930:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004608:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572705.090935:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572705.090937:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572705.090939:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.090942:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572705.090946:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572705.090947:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.090952:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572705.090955:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788572705.090964:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572705.090969:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004608, portal 10 00000100:00000001:0.0:1788572705.090971:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572705.090973:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788572705.090980:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.090987:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572705.090990:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572705.090994:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed315d8. 00000100:00000200:0.0:1788572705.091004:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810004608, offset 0 00000400:00000200:0.0:1788572705.091010:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091030:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091039:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572705.091046:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572705.091053:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572705.091060:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572705.091067:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091077:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540181:540181:254:4294967295] 192.168.202.148@tcp LPNI seq info [540146:540146:6:4294967295] 00000400:00000200:0.0:1788572705.091087:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091104:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572705.091114:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572705.091123:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788572705.091136:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572705.091152:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572705.091159:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572705.091168:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572705.091173:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307844125 00000100:00000001:0.0:1788572705.091179:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091184:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307844125 00000100:00000001:0.0:1788572705.091191:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091195:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572705.091199:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091205:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091210:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.091218:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.091221:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572705.091224:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.091226:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.091229:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572705.091234:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.091262:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.091266:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.091270:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572705.091274:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572705.091280:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.091288:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572705.091298:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572705.091301:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572705.091304:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.091308:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572705.091313:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572705.091317:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.091329:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572705.091332:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d68. 00000400:00000010:0.0:1788572705.091338:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788572705.091342:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004736, portal 4 00000100:00000001:0.0:1788572705.091344:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572705.091346:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291922176 : -109714417629440 : ffff9c371f1c4700) 00000100:00000040:0.0:1788572705.091355:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.091362:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572705.091365:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572705.091367:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31770. 00000100:00000200:0.0:1788572705.091369:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810004736, offset 0 00000400:00000200:0.0:1788572705.091374:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091378:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091381:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572705.091384:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572705.091387:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572705.091389:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572705.091392:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091403:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540182:540182:253:4294967295] 192.168.202.148@tcp LPNI seq info [540147:540147:5:4294967295] 00000400:00000200:0.0:1788572705.091408:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091416:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572705.091422:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572705.091426:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788572705.091430:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572705.091436:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572705.091440:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572705.091443:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572705.091445:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307844257 00000100:00000001:0.0:1788572705.091447:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091449:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307844257 00000100:00000001:0.0:1788572705.091451:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091453:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572705.091456:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091459:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091461:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.091465:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.091466:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572705.091468:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.091470:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.091471:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572705.091475:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.091483:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.091484:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.091487:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572705.091488:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572705.091491:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.091494:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004864:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572705.091506:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572705.091508:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572705.091510:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572705.091511:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572705.091515:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572705.091517:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572705.091520:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572705.091523:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788572705.091527:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788572705.091530:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004864, portal 4 00000100:00000001:0.0:1788572705.091532:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572705.091533:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788572705.091539:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572705.091546:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572705.091549:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572705.091551:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31198. 00000100:00000200:0.0:1788572705.091553:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810004864, offset 0 00000400:00000200:0.0:1788572705.091557:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091563:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091567:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572705.091569:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572705.091572:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572705.091574:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572705.091576:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572705.091579:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540183:540183:252:4294967295] 192.168.202.148@tcp LPNI seq info [540148:540148:4:4294967295] 00000400:00000200:0.0:1788572705.091583:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572705.091589:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572705.091593:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572705.091597:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788572705.091601:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788572705.091614:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572705.091618:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572705.091621:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572705.091624:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307844266 00000100:00000001:0.0:1788572705.091663:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091668:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307844267 00000100:00000001:0.0:1788572705.091671:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307847902 00000100:00000001:0.0:1788572705.091673:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572705.091676:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091682:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091685:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.091690:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.091693:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572705.091696:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.091698:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.091701:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.091703:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572708.162385:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572708.162389:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572708.162391:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572708.162399:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572705/real 1788572708] req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.162412:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.162414:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.162417:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788572708.162421:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31660 00000400:00000010:0.0:1788572708.162423:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31660. 00000100:00000001:0.0:1788572708.162426:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.162430:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.162440:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162459:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.162462:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.162464:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.162467:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.162472:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162480:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.162485:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162493:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572708.162496:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.162498:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.162500:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.162502:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572708.162505:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572708.162508:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572708.162513:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572708.162516:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572708.162522:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.162531:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004864:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572708.162536:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572708.162540:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572705/real 1788572708] req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.162549:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.162563:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.162571:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d68. 00000400:00000200:0.0:1788572708.162574:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31bb0 00000400:00000010:0.0:1788572708.162575:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31bb0. 00000100:00000001:0.0:1788572708.162578:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.162582:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.162591:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162598:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.162600:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.162603:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.162604:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.162608:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162615:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.162619:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.162626:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572708.162629:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.162631:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.162632:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163135:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572708.163138:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572708.163143:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572708.163148:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572708.163150:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572708.163156:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.163166:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572708.163178:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572708.163183:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572705/real 1788572708] req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.163193:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.163195:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.163197:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8210. 00000400:00000200:0.0:1788572708.163200:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788572708.163202:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788572708.163204:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.163209:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.163219:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163226:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.163228:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163231:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163232:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.163236:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163245:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.163249:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163257:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572708.163260:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163262:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163263:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163265:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572708.163266:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572708.163269:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572708.163276:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572708.163278:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572708.163283:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.163292:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572708.163297:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572708.163301:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572705/real 1788572708] req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.163310:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.163311:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.163313:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8f78. 00000400:00000200:0.0:1788572708.163316:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572708.163317:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572708.163320:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.163323:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.163332:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163339:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.163340:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163342:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163344:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.163348:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163355:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.163359:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163379:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572708.163382:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163384:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163385:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163387:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572708.163389:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572708.163391:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572708.163395:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572708.163397:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572708.163401:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.163410:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810004608:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572708.163414:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572708.163419:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572705/real 1788572708] req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.163427:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.163428:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.163431:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572708.163433:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572708.163434:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572708.163436:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.163440:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.163448:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163455:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.163456:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163462:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163464:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.163468:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163476:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.163480:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163488:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572708.163490:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163492:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.163493:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163495:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572708.163497:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572708.163499:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572708.163503:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572708.163525:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572708.163530:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.163538:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810004352:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572708.163542:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163545:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.163560:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810004864/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163568:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.163572:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.163577:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788572708.163581:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.163584:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.163586:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572708.163594:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.163597:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788572708.163601:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.163603:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.163605:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788572708.163608:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.163610:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163612:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.163617:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4700 x1875440810004736/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163624:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.163662:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.163665:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572708.163669:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.163671:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.163673:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572708.163676:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.163679:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233ec00. 02000000:00000001:0.0:1788572708.163682:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.163684:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.163686:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4700. 00000100:00000001:0.0:1788572708.163689:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.163690:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163693:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.163698:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440810004480/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163706:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.163707:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.163709:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572708.163711:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.163712:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.163713:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=14 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572708.163715:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.163717:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788572708.163719:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.163725:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.163727:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788572708.163729:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.163731:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163733:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.163737:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440810004608/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163744:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.163745:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.163747:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572708.163749:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.163751:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.163752:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572708.163754:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.163755:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788572708.163758:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.163759:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.163760:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788572708.163762:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.163764:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163765:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.163769:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440810004352/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.163775:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.163777:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.163778:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572708.163780:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.163782:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.163783:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572708.163785:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.163786:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788572708.163788:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.163789:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.163791:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788572708.163793:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.163794:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163800:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.163808:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572708.163810:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572708.163812:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572708.163814:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572708.929711:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307847902, at 4307847936 00000100:00000010:0.0:1788572708.929738:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788572709.057791:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572709.057796:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572709.057799:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572710.210590:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572710.210596:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572710.210598:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572710.210607:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.210617:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.210619:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.210623:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572710.210626:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572710.210629:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.210632:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810004992:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572710.210664:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572710.210669:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572710.210672:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.210675:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572710.210681:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572710.210683:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572710.210689:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572710.210692:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572710.210700:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572710.210706:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810004992, portal 25 00000100:00000001:0.0:1788572710.210709:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572710.210711:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788572710.210717:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.210732:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572710.210736:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572710.210739:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572710.210743:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810004992, offset 0 00000400:00000200:0.0:1788572710.210749:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572710.210757:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.210764:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572710.210769:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572710.210773:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572710.210778:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572710.210781:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572710.210785:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540184:540184:256:4294967295] 192.168.202.148@tcp LPNI seq info [540149:540149:8:4294967295] 00000400:00000200:0.0:1788572710.210790:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.210800:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572710.210806:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.210812:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572710.210818:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572710.210822:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1788572710.210837:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.210840:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572710.210845:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572710.210850:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572710.210856:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572710.210859:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572710.210864:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572710.210869:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.210880:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.210884:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.210905:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.210908:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572710.210914:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572710.210919:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572710.210922:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307853023 00000100:00000010:0.0:1788572710.210927:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572710.210933:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572710.210940:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307853023 00000100:00000010:0.0:1788572710.210943:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116269c0. 00000100:00000001:0.0:1788572710.210946:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572710.210950:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572710.210953:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.210956:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572710.210961:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572710/real 1788572710] req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572710.210972:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572710.210974:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572710.210977:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572710.210980:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572710.210982:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572710.210985:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572710.210988:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572710.210998:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211005:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572710.211007:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211009:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211012:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572710.211016:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211033:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572710.211039:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211047:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572710.211050:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211052:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211054:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211056:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572710.211058:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572710.211061:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572710.211065:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572710.211092:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572710.211099:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572710.211108:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810004992:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572710.211114:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572710.211118:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211136:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.211138:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211141:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572710.211143:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572710.211146:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.211149:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005120:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572710.211153:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572710.211155:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572710.211157:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211165:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572710.211169:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572710.211171:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572710.211176:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572710.211178:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572710.211182:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572710.211186:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810005120, portal 10 00000100:00000001:0.0:1788572710.211188:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572710.211190:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788572710.211197:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211205:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572710.211207:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572710.211209:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572710.211211:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810005120, offset 0 00000400:00000200:0.0:1788572710.211214:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211217:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211221:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572710.211224:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572710.211227:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572710.211229:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572710.211232:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211235:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540185:540185:256:4294967295] 192.168.202.148@tcp LPNI seq info [540150:540150:8:4294967295] 00000400:00000200:0.0:1788572710.211239:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211246:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572710.211251:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211255:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572710.211260:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572710.211263:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1788572710.211276:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211278:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572710.211286:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572710.211290:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572710.211292:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572710.211294:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572710.211296:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572710.211299:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211305:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.211307:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211312:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.211313:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572710.211315:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572710.211318:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572710.211320:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572710.211322:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307849241 00000100:00000001:0.0:1788572710.211324:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.211325:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307849241 00000100:00000001:0.0:1788572710.211327:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.211329:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572710.211330:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.211331:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572710.211334:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572710/real 1788572710] req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572710.211339:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572710.211340:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572710.211341:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572710.211342:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572710.211343:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572710.211345:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572710.211351:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572710.211356:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211360:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572710.211362:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211364:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211365:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572710.211369:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211375:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572710.211380:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211386:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572710.211389:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211391:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211392:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211394:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572710.211396:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572710.211399:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572710.211402:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572710.211404:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572710.211409:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572710.211416:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005120:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572710.211421:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572710.211424:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211444:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.211445:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211447:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572710.211449:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572710.211452:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.211455:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005248:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572710.211459:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572710.211461:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572710.211463:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211465:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572710.211475:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572710.211477:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572710.211481:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572710.211484:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572710.211488:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572710.211491:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810005248, portal 10 00000100:00000001:0.0:1788572710.211493:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572710.211495:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788572710.211515:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211523:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572710.211525:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572710.211528:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572710.211531:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810005248, offset 0 00000400:00000200:0.0:1788572710.211536:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211541:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211545:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572710.211548:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572710.211551:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572710.211553:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572710.211556:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211566:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540186:540186:256:4294967295] 192.168.202.148@tcp LPNI seq info [540151:540151:8:4294967295] 00000400:00000200:0.0:1788572710.211570:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211577:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572710.211583:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211587:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572710.211591:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572710.211594:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1788572710.211605:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211607:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572710.211610:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572710.211612:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572710.211615:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572710.211616:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572710.211618:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572710.211621:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211627:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.211629:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211633:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.211662:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572710.211667:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572710.211671:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572710.211675:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572710.211677:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307849246 00000100:00000001:0.0:1788572710.211680:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.211682:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307849246 00000100:00000001:0.0:1788572710.211685:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.211687:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572710.211696:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.211698:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572710.211702:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572710/real 1788572710] req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572710.211712:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572710.211713:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572710.211715:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572710.211718:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572710.211720:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572710.211722:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572710.211725:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572710.211733:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211741:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572710.211743:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211744:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211746:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572710.211749:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211756:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572710.211760:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211767:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572710.211769:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211771:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.211772:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.211774:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572710.211776:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572710.211778:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572710.211789:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572710.211791:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572710.211796:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572710.211804:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005248:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572710.211808:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572710.211812:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211818:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.211819:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211822:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572710.211823:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572710.211825:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.211828:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005376:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572710.211831:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572710.211833:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572710.211834:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572710.211836:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572710.211840:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572710.211841:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572710.211845:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572710.211848:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788572710.211851:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572710.211855:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810005376, portal 4 00000100:00000001:0.0:1788572710.211857:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572710.211858:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360000 : -109714422191616 : ffff9c371ed6aa00) 00000100:00000040:0.0:1788572710.211864:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211870:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572710.211878:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572710.211880:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788572710.211883:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810005376, offset 0 00000400:00000200:0.0:1788572710.211887:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211892:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211896:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572710.211899:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572710.211902:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572710.211904:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572710.211907:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572710.211911:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540187:540187:256:4294967295] 192.168.202.148@tcp LPNI seq info [540152:540152:8:4294967295] 00000400:00000200:0.0:1788572710.211915:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572710.211921:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572710.211925:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211929:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788572710.211933:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572710.211936:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1788572710.211948:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572710.211950:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0d00. 00000400:00000200:0.0:1788572710.211954:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572710.211959:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572710.211963:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788572710.211965:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788572710.211969:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572710.211974:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211983:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.211987:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.211995:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.211997:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572710.212009:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572710.212013:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572710.212017:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572710.212019:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307849378 00000100:00000001:0.0:1788572710.212021:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.212023:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307849378 00000100:00000001:0.0:1788572710.212025:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307853023 00000100:00000001:0.0:1788572710.212027:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572710.212029:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.212031:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572710.212034:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572710/real 1788572710] req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572710.212042:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572710.212043:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572710.212045:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8c08. 00000400:00000200:0.0:1788572710.212047:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572710.212049:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572710.212051:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572710.212054:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572710.212063:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212070:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572710.212071:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212073:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.212075:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572710.212078:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212085:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572710.212090:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212105:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572710.212107:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212109:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572710.212110:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212112:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572710.212114:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572710.212116:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572710.212121:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572710.212123:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572710.212128:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572710.212137:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810005376:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572710.212141:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212144:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.212147:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810004992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212154:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572710.212157:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572710.212160:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572710.212163:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572710.212166:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572710.212167:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572710.212169:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572710.212172:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788572710.212176:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572710.212177:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572710.212179:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788572710.212183:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572710.212184:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212192:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.212196:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a300 x1875440810005120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212202:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572710.212204:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572710.212206:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788572710.212208:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572710.212210:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572710.212211:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=14 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572710.212213:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572710.212215:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788572710.212217:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572710.212218:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572710.212220:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a300. 00000100:00000001:0.0:1788572710.212222:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572710.212223:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212225:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.212228:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810005248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212235:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572710.212236:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572710.212237:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572710.212240:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572710.212241:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572710.212242:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572710.212244:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572710.212245:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788572710.212248:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572710.212249:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572710.212250:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788572710.212252:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572710.212253:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212254:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572710.212257:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6aa00 x1875440810005376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572710.212275:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572710.212277:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572710.212278:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788572710.212281:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572710.212282:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572710.212283:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572710.212285:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572710.212286:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788572710.212288:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572710.212290:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572710.212291:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6aa00. 00000100:00000001:0.0:1788572710.212293:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572710.212294:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212297:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.212304:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572710.212306:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.212308:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572710.212310:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572714.049737:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307853023, at 4307853056 00000100:00000010:0.0:1788572714.049750:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788572714.177893:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572714.177901:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572714.177905:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572715.329820:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572715.329832:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572715.329835:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572715.329840:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572715.329845:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572715.329853:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13181 00000400:00000200:0.0:1788572715.329864:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572715.329874:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572715.329893:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572715.329897:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.329924:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.329928:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.329930:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572715.329935:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572715.329937:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572715.329940:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572715.329945:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572715.329950:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572715.329953:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572715.329959:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.329962:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.329964:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.329978:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572715.329982:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572715.329986:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572715.329989:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572715.329992:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572715.329996:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572715.330000:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330003:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572715.330006:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572715.330014:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572715.330018:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572715.330022:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572715.330024:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330027:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572715.330031:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572715.330036:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572715.330045:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572715.330055:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.330067:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810005632/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572715.330091:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572715.330096:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.330113:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810005632/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572715.330137:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572715.330141:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572715.330145:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572715.330146:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572715.330148:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572715.330151:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572715.330154:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13181 00000400:00000200:0.0:1788572715.330159:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572715.330162:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572715.330165:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572715.330168:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330170:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.330172:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330174:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572715.330177:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572715.330178:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572715.330180:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572715.330183:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572715.330185:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572715.330188:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572715.330191:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330193:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.330196:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330199:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572715.330209:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572715.330215:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330224:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572715.330228:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572715.330238:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572715.330245:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330248:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572715.330256:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572715.330279:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330289:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572715.330299:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572715.330310:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=14 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572715.330316:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572715.330321:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330332:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572715.330335:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572715.330341:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572715.330351:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572715.330359:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.330386:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810005760/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572715.330403:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572715.330412:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.330425:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810005760/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572715.330449:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572715.330454:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572715.330465:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572715.330468:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572715.330472:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572715.330486:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572715.330513:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13181 00000400:00000200:0.0:1788572715.330526:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572715.330531:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572715.330534:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572715.330538:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330541:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.330549:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330564:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572715.330574:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572715.330576:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572715.330579:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572715.330584:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572715.330589:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572715.330598:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572715.330607:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330610:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.330618:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330625:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572715.330631:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572715.330688:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330692:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572715.330695:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572715.330699:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572715.330702:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330705:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572715.330710:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572715.330721:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330727:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572715.330735:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572715.330743:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572715.330752:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572715.330764:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.330773:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572715.330780:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572715.330786:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572715.330799:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572715.330811:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.330830:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810005888/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572715.330846:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572715.330850:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.330862:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810005888/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572715.330885:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572715.330888:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572715.330891:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572715.330893:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572715.330895:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572715.330899:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572715.330907:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13181 00000400:00000200:0.0:1788572715.330921:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572715.330932:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572715.330940:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572715.330946:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330953:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.330957:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.330966:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572715.330973:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572715.330978:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572715.330985:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572715.330991:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572715.330998:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572715.331012:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572715.331032:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331039:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.331042:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331051:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572715.331060:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572715.331070:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331086:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572715.331093:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572715.331101:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572715.331112:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331123:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572715.331129:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572715.331137:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331149:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572715.331158:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572715.331162:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572715.331172:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572715.331176:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331180:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572715.331191:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572715.331198:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572715.331207:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572715.331216:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.331227:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810006016/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572715.331247:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572715.331257:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.331266:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810006016/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572715.331330:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572715.331341:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572715.331345:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572715.331346:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572715.331348:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572715.331352:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572715.331357:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13181 00000400:00000200:0.0:1788572715.331368:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572715.331396:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572715.331402:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572715.331413:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331422:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.331430:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331436:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572715.331442:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572715.331446:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572715.331451:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572715.331455:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572715.331463:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572715.331471:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572715.331478:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331485:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572715.331493:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331500:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572715.331510:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572715.331525:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572715.331538:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572715.331545:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572715.331552:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572715.331569:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331579:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572715.331593:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572715.331605:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331612:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572715.331620:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572715.331631:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572715.331669:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572715.331671:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572715.331675:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572715.331679:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572715.331685:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572715.331696:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572715.331706:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.331721:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810006144/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572715.331733:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572715.331739:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.331750:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810006144/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572715.331787:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572715.331800:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13192) 00000100:00000001:0.0:1788572716.865731:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572716.865742:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572716.865746:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572716.865750:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572716.865752:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572716.865757:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.865761:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572717.889706:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572717.889714:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572717.889720:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572717.889729:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572717.889736:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572717.889744:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572717.889752:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572718.913712:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572718.913729:0:2404:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572718.913744:0:2404:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572633/real 1788572633] req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572718.913774:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572718.913776:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572718.913780:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2f20. 00000400:00000200:0.0:1788572718.913785:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3cc0 00000400:00000010:0.0:1788572718.913788:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3cc0. 00000100:00000001:0.0:1788572718.913792:0:2404:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572718.913798:0:2404:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572718.913809:0:2404:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572718.913817:0:2404:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572718.913819:0:2404:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913822:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572718.913824:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572718.913829:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572718.913836:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572718.913838:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9c3707cce000 already not connected (conn 10, was 3): DISCONN 00000100:00000001:0.0:1788572718.913848:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572718.913850:0:2404:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913856:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572718.913857:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913859:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572718.913861:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572718.913864:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913866:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572718.913869:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572718.913874:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572718.913895:0:2404:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572718.913897:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913898:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572718.913899:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572718.913903:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572718.913909:0:2404:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:5fa795d2-07a0-4260-8bbc-bb3cde572406:2404:1875440809997056:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572718.913912:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572718.913915:0:2404:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572718.913917:0:2404:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7b80 x1875440809997056/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572718 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572718.913922:0:2404:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572718.913925:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572718.913928:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3711215000. 02000000:00000001:0.0:1788572718.913932:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572718.913935:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572718.913936:0:2404:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=11 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572718.913938:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572718.913940:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c37042e9300. 02000000:00000001:0.0:1788572718.913943:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572718.913944:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572718.913946:0:2404:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7b80. 00000100:00000001:0.0:1788572718.913949:0:2404:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572718.913950:0:2404:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.913952:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572719.169775:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307858143, at 4307858176 00000100:00000010:0.0:1788572719.169797:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215000. 00000020:00000001:0.0:1788572719.297897:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572719.297903:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572719.297907:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572719.937703:0:2404:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572719.937715:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572719.937726:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572719.937732:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572719.937741:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572719.937754:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572719.937761:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572720.898259:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572720.898264:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572720.898266:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572720.898269:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572720.898272:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572720.898275:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13192 00000400:00000200:0.0:1788572720.898281:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572720.898286:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572720.898289:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572720.898291:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572720.898293:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572720.898295:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572720.898297:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572720.898299:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572720.898300:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572720.898301:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572720.898303:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572720.898305:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572720.898306:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572720.898308:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572720.898309:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572720.898311:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572720.898312:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572720.898315:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572720.898317:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572720.898325:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572720.898327:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572720.898330:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572720.898331:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572720.898333:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572720.898335:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572720.898338:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572720.898340:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572720.898342:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572720.898343:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572720.898345:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572720.898346:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572720.898349:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572720.898352:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572720.898355:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572720.898360:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810007040/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572720.898366:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572720.898368:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572720.898372:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810007040/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572720.898391:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572720.898396:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13197) 00000800:00000200:0.0:1788572721.409925:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788572721.985735:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572721.985741:0:2403:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572721.985756:0:2403:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572636/real 1788572636] req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572721.985776:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572721.985778:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572721.985782:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572721.985786:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572721.985789:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572721.985805:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572721.985810:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572721.985821:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572721.985829:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572721.985831:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985834:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572721.985836:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572721.985841:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572721.985849:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572721.985852:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9c3707cce000 already not connected (conn 11, was 3): DISCONN 00000100:00000001:0.0:1788572721.985856:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572721.985858:0:2403:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985862:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572721.985864:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985866:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572721.985868:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572721.985870:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985872:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572721.985874:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572721.985879:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572721.985887:0:2403:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572721.985891:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985892:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572721.985893:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572721.985898:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572721.985906:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809998080:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572721.985916:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572721.985919:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572721.985923:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440809998080/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572721 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572721.985931:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572721.985934:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572721.985938:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3704de6200. 02000000:00000001:0.0:1788572721.985941:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572721.985944:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572721.985946:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=9 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572721.985949:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572721.985952:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0500. 02000000:00000001:0.0:1788572721.985955:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572721.985957:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572721.985959:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788572721.985962:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572721.985964:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.985966:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572723.009788:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572723.009801:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572723.009805:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572723.009811:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572723.009815:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572723.009819:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572723.009823:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572724.033773:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572724.033790:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572724.033795:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572724.033801:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572724.033806:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572724.033811:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572724.033817:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572724.417852:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572724.417855:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572724.417858:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572724.737695:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307863711, at 4307863744 00000100:00000010:0.0:1788572724.737724:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322157e0. 00000100:00000001:0.0:1788572725.057714:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572725.057724:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572725.057727:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572725.057731:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572725.057734:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572725.057738:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572725.057743:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572726.081724:0:2403:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572726.081730:0:2403:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572726.081745:0:2403:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572641/real 1788572641] req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572726.081774:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572726.081779:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572726.081785:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37319f2000. 00000400:00000200:0.0:1788572726.081794:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f33b8 00000400:00000010:0.0:1788572726.081801:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f33b8. 00000100:00000001:0.0:1788572726.081810:0:2403:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572726.081821:0:2403:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572726.081842:0:2403:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572726.081850:0:2403:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572726.081853:0:2403:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.081856:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572726.081861:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572726.081871:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572726.081888:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572726.081893:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9c3707cce000 already not connected (conn 11, was 3): DISCONN 00000100:00000001:0.0:1788572726.081899:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572726.081903:0:2403:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.081926:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572726.081930:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572726.081933:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572726.081936:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572726.081938:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.081943:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572726.081945:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572726.081950:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572726.081959:0:2403:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572726.081965:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.081967:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572726.081968:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572726.081974:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572726.082003:0:2403:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:5fa795d2-07a0-4260-8bbc-bb3cde572406:2403:1875440809998720:192.168.202.148@tcp:400:kworker.0 00000100:00000001:0.0:1788572726.082013:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.082019:0:2403:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572726.082028:0:2403:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440809998720/t0(0) o400->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572726 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572726.082039:0:2403:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572726.082043:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572726.082047:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9c3711214600. 02000000:00000001:0.0:1788572726.082054:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572726.082058:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572726.082061:0:2403:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=6 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572726.082067:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572726.082070:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9c36010e0900. 02000000:00000001:0.0:1788572726.082074:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572726.082075:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572726.082080:0:2403:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788572726.082084:0:2403:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572726.082093:0:2403:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.082098:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.082119:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572726.082123:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572726.082126:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572726.082141:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572728.513792:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572728.513797:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572728.513801:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572729.537845:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572729.537850:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572729.537854:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572729.921798:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307868895, at 4307868928 00000100:00000010:0.0:1788572729.921828:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000100:00000001:0.0:1788572731.202065:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572731.202069:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572731.202073:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572731.202084:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202106:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.202110:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.202115:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572731.202120:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572731.202132:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.202141:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810007808:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572731.202146:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572731.202151:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572731.202155:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.202159:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572731.202164:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.202167:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.202176:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572731.202180:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572731.202187:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572731.202194:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810007808, portal 25 00000100:00000001:0.0:1788572731.202204:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572731.202207:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286338816 : -109714423212800 : ffff9c371ec71500) 00000100:00000040:0.0:1788572731.202213:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202223:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572731.202231:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572731.202235:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572731.202239:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810007808, offset 0 00000400:00000200:0.0:1788572731.202246:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572731.202257:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.202266:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572731.202272:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572731.202278:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572731.202283:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572731.202286:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572731.202292:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540205:540205:256:4294967295] 192.168.202.148@tcp LPNI seq info [540170:540170:8:4294967295] 00000400:00000200:0.0:1788572731.202299:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.202315:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572731.202325:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.202333:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572731.202342:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572731.202346:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 16, 4 secs later) 00000800:00000100:0.0:1788572731.202368:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.202373:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572731.202379:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572731.202387:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572731.202394:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572731.202397:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572731.202401:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572731.202408:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202444:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.202449:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202460:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.202463:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572731.202469:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.202473:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572731.202478:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4307874014 00000100:00000010:0.0:1788572731.202485:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788572731.202492:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572731.202501:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4307874014 00000100:00000010:0.0:1788572731.202505:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626c60. 00000100:00000001:0.0:1788572731.202511:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572731.202518:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572731.202523:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.202528:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572731.202535:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572731/real 1788572731] req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572731.202553:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572731.202556:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572731.202560:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572731.202564:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572731.202568:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572731.202572:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572731.202578:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572731.202598:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202607:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572731.202615:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202618:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.202620:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572731.202624:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202632:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572731.202660:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202676:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572731.202681:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202683:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.202685:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202690:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572731.202692:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572731.202696:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572731.202703:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572731.202715:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572731.202722:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572731.202736:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810007808:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788572731.202744:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202747:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.202754:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71500 x1875440810007808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202762:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572731.202765:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572731.202768:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.202771:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572731.202776:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572731.202782:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572731.202786:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572731.202789:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788572731.202794:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572731.202796:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572731.202800:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71500. 00000100:00000001:0.0:1788572731.202803:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572731.202806:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202812:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.202818:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572731.202822:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.202827:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572731.202830:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572731.202833:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572731.202843:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202862:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.202866:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.202870:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572731.202871:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572731.202873:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.202879:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810007936:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572731.202884:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572731.202887:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572731.202889:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.202892:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572731.202897:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.202901:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.202907:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572731.202911:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572731.202916:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572731.202921:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810007936, portal 10 00000100:00000001:0.0:1788572731.202923:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572731.202927:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788572731.202936:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.202952:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572731.202957:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572731.202961:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572731.202965:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810007936, offset 0 00000400:00000200:0.0:1788572731.202971:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572731.202990:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.202995:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572731.203000:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572731.203006:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572731.203011:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572731.203018:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203024:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540206:540206:256:4294967295] 192.168.202.148@tcp LPNI seq info [540171:540171:8:4294967295] 00000400:00000200:0.0:1788572731.203031:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203042:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572731.203051:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203056:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572731.203065:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572731.203071:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 16, 4 secs later) 00000800:00000100:0.0:1788572731.203088:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203092:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572731.203095:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572731.203101:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572731.203105:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572731.203109:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572731.203111:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572731.203118:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203135:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203142:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203171:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203173:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572731.203179:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572731.203182:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788572731.203185:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572731.203187:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307870233 00000100:00000001:0.0:1788572731.203189:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203190:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307870233 00000100:00000001:0.0:1788572731.203192:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203193:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572731.203195:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203196:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572731.203198:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572731/real 1788572731] req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572731.203204:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572731.203204:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572731.203206:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572731.203207:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572731.203208:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572731.203210:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572731.203212:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572731.203217:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203221:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572731.203222:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203223:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203224:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572731.203227:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203234:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572731.203238:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203242:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572731.203243:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203244:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203245:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203246:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572731.203247:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572731.203249:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572731.203251:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572731.203252:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572731.203255:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572731.203260:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810007936:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572731.203262:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572731.203265:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203268:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.203269:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203271:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572731.203272:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572731.203273:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.203275:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008064:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572731.203277:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572731.203278:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572731.203282:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203283:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572731.203285:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572731.203287:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.203289:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572731.203291:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572731.203293:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572731.203295:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810008064, portal 10 00000100:00000001:0.0:1788572731.203296:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572731.203297:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345984 : -109714423205632 : ffff9c371ec73100) 00000100:00000040:0.0:1788572731.203301:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203306:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572731.203308:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572731.203309:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572731.203311:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810008064, offset 0 00000400:00000200:0.0:1788572731.203313:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203316:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203318:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572731.203320:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572731.203321:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572731.203323:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572731.203325:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203327:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540207:540207:256:4294967295] 192.168.202.148@tcp LPNI seq info [540172:540172:8:4294967295] 00000400:00000200:0.0:1788572731.203329:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203334:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572731.203336:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203338:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572731.203341:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572731.203343:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 16, 4 secs later) 00000800:00000100:0.0:1788572731.203350:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203354:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572731.203356:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572731.203358:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572731.203361:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572731.203362:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572731.203364:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572731.203366:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203371:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203373:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203377:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203378:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572731.203380:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572731.203383:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788572731.203385:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572731.203386:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307870237 00000100:00000001:0.0:1788572731.203388:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203389:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307870237 00000100:00000001:0.0:1788572731.203391:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203392:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572731.203393:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203394:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572731.203396:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572731/real 1788572731] req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572731.203401:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572731.203402:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572731.203403:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572731.203404:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572731.203406:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572731.203410:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572731.203412:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572731.203418:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203422:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572731.203423:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203425:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203426:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572731.203428:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203432:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572731.203434:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203438:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572731.203439:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203440:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203441:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203442:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572731.203443:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572731.203445:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572731.203446:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572731.203448:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572731.203450:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572731.203454:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008064:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788572731.203457:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203461:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203463:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810007936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203468:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572731.203469:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572731.203470:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.203472:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572731.203473:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572731.203474:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572731.203475:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572731.203476:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788572731.203478:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572731.203479:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572731.203480:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788572731.203482:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572731.203483:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203484:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203486:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73100 x1875440810008064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203489:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572731.203490:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572731.203491:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572731.203492:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572731.203493:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572731.203493:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788572731.203494:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572731.203495:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788572731.203496:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572731.203497:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572731.203498:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73100. 00000100:00000001:0.0:1788572731.203499:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572731.203500:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203501:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203504:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572731.203505:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203506:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572731.203510:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572731.203511:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572731.203514:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203518:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.203519:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203521:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572731.203522:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572731.203523:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.203525:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008192:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572731.203527:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572731.203528:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572731.203529:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203530:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572731.203531:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572731.203533:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.203535:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572731.203537:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572731.203539:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572731.203541:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810008192, portal 4 00000100:00000001:0.0:1788572731.203543:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572731.203544:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286347776 : -109714423203840 : ffff9c371ec73800) 00000100:00000040:0.0:1788572731.203547:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203552:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572731.203553:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572731.203554:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572731.203556:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810008192, offset 0 00000400:00000200:0.0:1788572731.203558:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203561:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203563:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572731.203565:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572731.203566:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572731.203571:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572731.203573:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203575:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540208:540208:256:4294967295] 192.168.202.148@tcp LPNI seq info [540173:540173:8:4294967295] 00000400:00000200:0.0:1788572731.203578:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203582:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572731.203585:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203587:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572731.203590:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572731.203591:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 16, 4 secs later) 00000800:00000100:0.0:1788572731.203598:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203600:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572731.203601:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572731.203604:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572731.203606:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572731.203607:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572731.203608:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572731.203611:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203616:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203618:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203622:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203623:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572731.203625:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572731.203627:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788572731.203629:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572731.203630:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307870369 00000100:00000001:0.0:1788572731.203632:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203633:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307870369 00000100:00000001:0.0:1788572731.203655:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203657:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572731.203659:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203661:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572731.203665:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572731/real 1788572731] req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572731.203675:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572731.203676:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572731.203678:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572731.203680:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572731.203682:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572731.203684:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572731.203687:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572731.203697:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203704:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572731.203705:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203707:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203709:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572731.203712:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203719:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572731.203724:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203731:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572731.203734:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203735:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203737:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203744:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572731.203746:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572731.203749:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572731.203753:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572731.203755:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572731.203758:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572731.203763:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008192:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572731.203766:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572731.203768:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203773:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.203774:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203775:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572731.203777:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572731.203778:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.203779:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008320:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572731.203781:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572731.203783:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572731.203784:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572731.203785:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572731.203787:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.203789:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572731.203791:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788572731.203793:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788572731.203795:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788572731.203798:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810008320, portal 4 00000100:00000001:0.0:1788572731.203799:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572731.203800:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286336128 : -109714423215488 : ffff9c371ec70a80) 00000100:00000040:0.0:1788572731.203804:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203813:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572731.203815:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788572731.203817:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788572731.203820:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810008320, offset 0 00000400:00000200:0.0:1788572731.203824:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203832:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788572731.203835:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572731.203838:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788572731.203840:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572731.203842:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788572731.203846:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540209:540209:256:4294967295] 192.168.202.148@tcp LPNI seq info [540174:540174:8:4294967295] 00000400:00000200:0.0:1788572731.203850:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572731.203857:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788572731.203862:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203866:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788572731.203870:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788572731.203873:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 16, 4 secs later) 00000800:00000100:0.0:1788572731.203881:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788572731.203882:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788572731.203884:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572731.203886:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572731.203889:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788572731.203890:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788572731.203892:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572731.203894:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203900:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.203902:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203914:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203915:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572731.203918:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572731.203920:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788572731.203923:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788572731.203925:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4307870379 00000100:00000001:0.0:1788572731.203926:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203928:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4307870379 00000100:00000001:0.0:1788572731.203929:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4307874014 00000100:00000001:0.0:1788572731.203930:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572731.203932:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.203933:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572731.203935:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572731/real 1788572731] req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572731.203940:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572731.203941:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572731.203942:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788572731.203944:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788572731.203945:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788572731.203946:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572731.203948:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572731.203954:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203958:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572731.203958:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203959:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203960:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572731.203962:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203970:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572731.203973:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.203992:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572731.203994:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203995:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572731.203996:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.203997:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572731.203998:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572731.204000:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572731.204003:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788572731.204004:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572731.204007:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572731.204012:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810008320:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788572731.204015:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.204016:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.204018:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73800 x1875440810008192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.204022:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572731.204023:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572731.204024:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788572731.204026:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572731.204028:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572731.204029:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=2 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788572731.204030:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572731.204031:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788572731.204033:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572731.204037:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572731.204038:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73800. 00000100:00000001:0.0:1788572731.204040:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572731.204041:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.204042:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572731.204044:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70a80 x1875440810008320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572731.204049:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572731.204050:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572731.204051:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788572731.204052:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572731.204053:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572731.204053:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788572731.204054:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572731.204055:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788572731.204056:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572731.204057:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572731.204058:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70a80. 00000100:00000001:0.0:1788572731.204059:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572731.204060:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.204061:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.204064:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572731.204064:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.204066:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572731.204066:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572733.633869:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572733.633873:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572733.633877:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572734.657853:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572734.657858:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572734.657862:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572735.041717:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307874014, at 4307874048 00000100:00000010:0.0:1788572735.041732:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00080000:0.0:1788572736.321746:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572736.321756:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572736.321773:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572736.321777:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572736.321782:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572736.321789:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13202 00000400:00000200:0.0:1788572736.321798:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572736.321815:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572736.321818:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572736.321821:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.321825:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.321828:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.321831:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572736.321835:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572736.321837:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572736.321840:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572736.321843:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572736.321846:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572736.321860:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572736.321864:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.321867:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.321869:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.321871:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572736.321874:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572736.321878:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572736.321881:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572736.321884:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572736.321888:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572736.321891:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.321894:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572736.321896:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572736.321903:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572736.321907:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572736.321918:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572736.321920:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.321923:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572736.321926:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572736.321930:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572736.321937:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572736.321943:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572736.321952:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810008448/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572736.321963:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572736.321968:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.321973:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810008448/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572736.321996:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572736.322008:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572736.322011:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572736.322013:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572736.322015:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572736.322019:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572736.322022:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13202 00000400:00000200:0.0:1788572736.322026:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572736.322029:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572736.322031:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572736.322036:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322039:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322043:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322048:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572736.322050:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572736.322051:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572736.322053:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572736.322056:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572736.322065:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572736.322069:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572736.322077:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322081:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322085:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322089:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572736.322092:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572736.322096:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322102:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572736.322105:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572736.322112:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572736.322117:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322118:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572736.322122:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572736.322129:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322133:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572736.322138:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572736.322144:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572736.322152:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572736.322156:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322162:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572736.322167:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572736.322171:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572736.322180:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572736.322187:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572736.322199:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810008576/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572736.322216:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572736.322220:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.322232:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810008576/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572736.322276:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572736.322282:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572736.322289:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572736.322291:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572736.322293:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572736.322300:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572736.322307:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13202 00000400:00000200:0.0:1788572736.322318:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572736.322326:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572736.322330:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572736.322338:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322351:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322355:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322357:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572736.322359:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572736.322360:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572736.322361:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572736.322363:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572736.322365:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572736.322367:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572736.322374:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322376:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322380:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322384:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572736.322386:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572736.322391:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322393:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572736.322401:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572736.322405:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572736.322409:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322411:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572736.322415:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572736.322421:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322425:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572736.322433:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572736.322440:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572736.322447:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572736.322459:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322463:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572736.322466:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572736.322470:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572736.322476:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572736.322482:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572736.322492:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810008704/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572736.322516:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572736.322521:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.322525:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810008704/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572736.322536:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572736.322539:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572736.322546:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572736.322548:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572736.322551:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572736.322556:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572736.322561:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13202 00000400:00000200:0.0:1788572736.322573:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572736.322580:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572736.322582:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572736.322595:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322597:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322599:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322605:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572736.322606:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572736.322608:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572736.322611:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572736.322615:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572736.322617:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572736.322621:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572736.322628:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322633:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322680:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322686:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572736.322693:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572736.322701:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322708:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572736.322715:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572736.322720:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572736.322727:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322733:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572736.322738:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572736.322745:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322751:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572736.322760:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572736.322765:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572736.322770:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572736.322773:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.322777:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572736.322780:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572736.322784:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572736.322791:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572736.322806:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572736.322822:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810008832/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572736.322840:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572736.322845:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.322865:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810008832/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572736.322882:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572736.322888:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572736.322895:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572736.322897:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572736.322902:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572736.322909:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572736.322916:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13202 00000400:00000200:0.0:1788572736.322927:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572736.322936:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572736.322943:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572736.322950:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322957:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.322964:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.322971:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572736.322977:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572736.322983:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572736.322987:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572736.322993:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572736.322998:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572736.323001:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572736.323006:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.323009:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572736.323025:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572736.323029:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572736.323036:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572736.323044:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572736.323050:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572736.323055:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572736.323060:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572736.323066:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.323071:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572736.323075:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572736.323081:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.323085:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572736.323090:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572736.323096:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572736.323101:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572736.323105:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572736.323110:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572736.323114:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572736.323119:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572736.323127:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572736.323136:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572736.323147:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810008960/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572736.323167:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572736.323172:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.323181:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810008960/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572736.323199:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572736.323206:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13213) 00000020:00000001:0.0:1788572738.753935:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572738.753941:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572738.753945:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572739.777995:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572739.777999:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572739.778002:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572740.161790:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307879134, at 4307879168 00000100:00000010:0.0:1788572740.161811:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 10000000:00000001:0.0:1788572740.929836:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788572740.929861:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a95c0. 00000020:00000010:0.0:1788572740.929872:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de6200. 00000020:00000010:0.0:1788572740.929884:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788572740.929890:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788572740.929897:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788572740.929905:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788572740.929911:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626180. 00040000:00000010:0.0:1788572740.929917:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788572740.929924:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3713ce0400. 10000000:00000040:0.0:1788572740.929931:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788572740.929935:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788572740.929941:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788572740.929954:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788572740.929958:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788572740.929963:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788572740.929967:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626180. 00040000:00000010:0.0:1788572740.929971:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788572740.929975:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3713ce0400. 00000020:00000010:0.0:1788572740.929991:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de6200. 10000000:00000010:0.0:1788572740.929994:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a95c0. 10000000:01000000:0.0:1788572740.929999:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788572740.930003:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788572740.934320:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788572740.934325:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788572740.934328:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788572740.934331:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788572740.934333:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788572740.934345:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788572740.934346:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788572740.934349:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788572740.934351:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788572740.934353:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788572740.934356:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788572742.849765:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788572742.849774:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788572742.849783:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788572742.849787:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788572742.849792:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788572742.849795:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788572742.849798:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788572742.849803:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788572742.849808:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788572742.849814:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788572742.849818:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788572742.849830:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572742.849834:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788572742.849839:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572742.849842:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572742.849846:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572742.849854:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3713ce0400. 00000100:00000001:0.0:1788572742.849861:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572742.849864:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788572742.849867:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788572742.849873:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46900. 00010000:00000001:0.0:1788572742.849877:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788572742.849883:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788572742.849889:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788572742.849892:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x14415860931689fc to hash 00000020:00000001:0.0:1788572742.849895:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788572742.849896:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572742.849899:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572742.849901:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572742.849904:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572742.849918:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572742.849926:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572742.849935:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788572742.849939:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788572742.849941:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788572742.849946:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788572742.849947:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611584 : -109714523940032 : ffff9c3718c61b40) 00000100:00000001:0.0:1788572742.849949:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572742.849950:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788572742.849954:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572742.849957:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572742.849959:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788572742.849960:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572742.849969:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572742.849975:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788572742.849978:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572742.849983:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572742.849986:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572742.849990:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788572742.849992:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572742.849993:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572742.849994:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61b40 going to sleep for 0 seconds 00000100:00000001:0.0:1788572742.849996:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572742.850001:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788572742.850004:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572742.850006:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572742.850007:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572742.850010:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572742.850016:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810009728:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788572742.850019:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572742.850021:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788572742.850023:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788572742.850024:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572742.850026:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572742.850031:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572742.850033:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788572742.850037:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788572742.850038:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788572742.850040:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788572742.850042:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572742.850044:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572742.850046:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788572742.850047:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572742.850049:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00010000:0.0:1788572742.850052:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850056:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788572742.850058:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850069:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788572742.850072:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850076:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572742.850077:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788572742.850079:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850082:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572742.850083:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572742.850085:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788572742.850087:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788572742.850091:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788572742.850093:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788572742.850094:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850098:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788572742.850099:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788572742.850100:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572742.850101:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=5 00000020:00000001:0.0:1788572742.850102:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572742.850104:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788572742.850106:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572742.850108:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788572742.850112:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788572742.850117:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788572742.850121:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788572742.850123:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788572742.850124:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572742.850126:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788572742.850136:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572742.850138:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572742.850139:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572742.850141:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572742.850142:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572742.850146:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30400 with handle 0x14415860931689fc from hash 00010000:00000001:0.0:1788572742.850148:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572742.850150:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572742.850151:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572742.850151:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572742.850153:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572742.850154:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788572742.850155:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572742.850156:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572742.850157:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788572742.850158:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572742.850160:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788572742.850164:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572742.850164:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572742.850165:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788572742.850166:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788572742.850167:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572742.850168:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572742.850169:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572742.850171:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x14415860931689fc lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788572742.850176:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46900 count: 0 00010000:00000001:0.0:1788572742.850186:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572742.850187:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788572742.850189:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572742.850192:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440810009728/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572742.850197:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788572742.850199:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572742.850200:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788572742.850201:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572742.850203:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3713ce0400. 02000000:00000001:0.0:1788572742.850206:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572742.850208:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572742.850209:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788572742.850213:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572742.850213:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788572742.850215:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788572742.850217:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788572742.850219:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788572743.873859:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572743.873864:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572743.873868:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572744.897828:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572744.897833:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572744.897837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572746.689809:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307885663, at 4307885696 00000100:00000010:0.0:1788572746.689826:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788572748.993851:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572748.993857:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572748.993861:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572750.017973:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572750.017980:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572750.017984:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572751.937756:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307890910, at 4307890944 00000100:00000010:0.0:1788572751.937774:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000100:00080000:0.0:1788572753.217793:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572753.217828:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572753.217830:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572753.217835:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572753.217840:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572753.217848:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13219 00000400:00000200:0.0:1788572753.217857:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572753.217864:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572753.217867:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572753.217870:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.217873:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.217875:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.217877:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572753.217880:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572753.217882:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572753.217885:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572753.217889:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572753.217890:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572753.217892:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572753.217894:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.217895:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.217897:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.217898:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572753.217901:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572753.217904:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572753.217906:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572753.217909:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572753.217913:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572753.217916:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.217919:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572753.217921:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572753.217930:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572753.217949:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572753.217955:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572753.217958:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.217962:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572753.217966:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572753.217972:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572753.217982:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572753.217993:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572753.218006:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810011136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572753.218017:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572753.218023:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.218028:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810011136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572753.218055:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572753.218059:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572753.218064:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572753.218065:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572753.218067:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572753.218071:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572753.218075:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13219 00000400:00000200:0.0:1788572753.218081:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572753.218085:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572753.218088:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572753.218095:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218102:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218109:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218114:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572753.218122:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572753.218126:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572753.218131:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572753.218147:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572753.218154:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572753.218160:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572753.218171:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218178:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218183:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218189:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572753.218193:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572753.218202:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218208:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572753.218211:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572753.218222:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572753.218229:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218236:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572753.218245:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572753.218252:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218264:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572753.218270:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572753.218274:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572753.218277:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572753.218280:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218283:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572753.218285:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572753.218287:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572753.218293:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572753.218300:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572753.218313:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810011264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572753.218334:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572753.218336:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.218340:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810011264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572753.218375:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572753.218379:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572753.218386:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572753.218389:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572753.218392:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572753.218395:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572753.218400:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13219 00000400:00000200:0.0:1788572753.218409:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572753.218417:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572753.218422:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572753.218429:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218432:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218436:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218439:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572753.218440:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572753.218441:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572753.218444:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572753.218447:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572753.218450:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572753.218454:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572753.218460:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218463:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218464:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218465:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572753.218467:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572753.218470:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218473:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572753.218482:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572753.218486:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572753.218489:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218491:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572753.218496:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572753.218501:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218505:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572753.218510:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572753.218514:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572753.218516:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572753.218520:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218524:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572753.218526:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572753.218529:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572753.218534:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572753.218541:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572753.218549:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810011392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572753.218556:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572753.218558:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.218564:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810011392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572753.218575:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572753.218595:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572753.218601:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572753.218603:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572753.218605:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572753.218608:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572753.218612:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13219 00000400:00000200:0.0:1788572753.218618:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572753.218624:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572753.218627:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572753.218675:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218679:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218685:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218690:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572753.218696:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572753.218698:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572753.218700:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572753.218705:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572753.218708:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572753.218713:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572753.218772:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218774:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218775:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218778:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572753.218783:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572753.218789:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218793:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572753.218799:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572753.218803:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572753.218806:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218808:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572753.218810:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572753.218812:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218813:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572753.218817:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572753.218819:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572753.218823:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572753.218827:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.218831:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572753.218834:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572753.218837:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572753.218843:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572753.218856:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572753.218866:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810011520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572753.218874:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572753.218876:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.218879:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810011520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572753.218887:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572753.218891:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572753.218897:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572753.218900:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572753.218903:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572753.218908:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572753.218912:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13219 00000400:00000200:0.0:1788572753.218921:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572753.218926:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572753.218933:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572753.218936:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218940:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218943:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218944:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572753.218946:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572753.218947:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572753.218951:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572753.218953:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572753.218958:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572753.218960:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572753.218965:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218969:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572753.218979:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572753.218983:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572753.218988:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572753.218994:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572753.219000:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572753.219003:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572753.219008:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572753.219012:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.219017:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572753.219021:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572753.219025:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.219031:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572753.219036:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572753.219043:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572753.219047:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572753.219051:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572753.219055:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572753.219058:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572753.219062:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572753.219070:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572753.219077:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572753.219084:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810011648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572753.219099:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572753.219105:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.219117:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810011648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572753.219143:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572753.219153:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13229) 00000020:00000001:0.0:1788572754.113828:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572754.113834:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572754.113838:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572755.137863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572755.137867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572755.137871:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572757.057739:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307896030, at 4307896064 00000100:00000010:0.0:1788572757.057758:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788572759.233844:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572759.233847:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572759.233850:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572759.297882:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788572760.257888:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572760.257892:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572760.257895:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572762.177737:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307901150, at 4307901184 00000100:00000010:0.0:1788572762.177755:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000020:00000001:0.0:1788572764.353773:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572764.353777:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572764.353781:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572764.930286:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572764.930293:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572764.930296:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572764.930300:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572764.930304:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572764.930308:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13235 00000400:00000200:0.0:1788572764.930318:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572764.930325:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572764.930328:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572764.930331:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930334:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930337:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930340:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572764.930343:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572764.930345:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572764.930357:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572764.930360:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572764.930362:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572764.930365:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572764.930369:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930371:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930373:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930376:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572764.930379:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572764.930382:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930386:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572764.930388:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572764.930391:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572764.930394:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930396:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572764.930399:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572764.930404:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572764.930407:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572764.930409:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572764.930411:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930414:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572764.930416:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572764.930419:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572764.930424:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572764.930429:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.930436:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810013184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572764.930445:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572764.930448:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.930453:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810013184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572764.930479:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572764.930489:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572764.930493:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572764.930494:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572764.930496:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572764.930499:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572764.930503:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13235 00000400:00000200:0.0:1788572764.930507:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572764.930510:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572764.930513:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572764.930516:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930519:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930521:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930523:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572764.930526:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572764.930527:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572764.930529:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572764.930532:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572764.930535:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572764.930538:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572764.930542:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930545:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930548:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930550:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572764.930554:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572764.930561:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930568:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572764.930572:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572764.930581:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572764.930588:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930594:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572764.930621:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572764.930628:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930668:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572764.930678:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572764.930685:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572764.930694:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572764.930697:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930701:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572764.930704:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572764.930710:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572764.930721:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572764.930729:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.930742:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810013312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572764.930759:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572764.930764:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.930772:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810013312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572764.930793:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572764.930800:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572764.930808:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572764.930813:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572764.930817:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572764.930834:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572764.930845:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13235 00000400:00000200:0.0:1788572764.930859:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572764.930865:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572764.930869:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572764.930873:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930878:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930883:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930895:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572764.930901:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572764.930904:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572764.930915:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572764.930921:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572764.930926:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572764.930930:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572764.930936:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930941:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.930947:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930952:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572764.930958:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572764.930965:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572764.930971:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572764.930975:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572764.930981:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572764.930987:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.930991:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572764.930994:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572764.931001:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931005:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572764.931013:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572764.931018:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572764.931024:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572764.931026:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931030:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572764.931033:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572764.931037:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572764.931045:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572764.931051:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.931062:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810013440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572764.931095:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572764.931100:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.931109:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810013440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572764.931129:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572764.931136:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572764.931146:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572764.931150:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572764.931154:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572764.931162:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572764.931169:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13235 00000400:00000200:0.0:1788572764.931178:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572764.931188:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572764.931195:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572764.931204:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931210:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.931218:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931225:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572764.931232:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572764.931237:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572764.931243:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572764.931249:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572764.931256:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572764.931263:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572764.931272:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931275:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.931277:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931280:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572764.931296:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572764.931307:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931316:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572764.931321:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572764.931327:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572764.931333:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931339:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572764.931344:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572764.931352:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931358:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572764.931367:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572764.931374:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572764.931379:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572764.931383:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931390:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572764.931394:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572764.931399:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572764.931407:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572764.931416:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.931429:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810013568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572764.931452:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572764.931457:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.931466:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810013568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572764.931485:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572764.931491:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572764.931499:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572764.931504:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572764.931509:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572764.931516:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572764.931532:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13235 00000400:00000200:0.0:1788572764.931554:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572764.931565:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572764.931573:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572764.931584:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931591:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.931613:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931619:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572764.931627:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572764.931631:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572764.931662:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572764.931667:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572764.931672:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572764.931678:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572764.931690:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931697:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572764.931704:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931711:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572764.931719:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572764.931728:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572764.931737:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572764.931746:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572764.931751:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572764.931758:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931763:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572764.931769:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572764.931775:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931779:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572764.931787:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572764.931794:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572764.931802:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572764.931815:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.931821:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572764.931825:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572764.931828:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572764.931837:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572764.931845:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.931858:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810013696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572764.931890:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572764.931897:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.931908:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810013696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572764.931946:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572764.931954:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13241) 00000020:00000001:0.0:1788572765.377838:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572765.377841:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572765.377844:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572768.769716:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307907743, at 4307907776 00000100:00000010:0.0:1788572768.769730:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788572769.473875:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572769.473878:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572769.473880:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572770.113728:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572770.113740:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572770.113742:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572770.113746:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572770.113749:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572770.113755:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13236 00000400:00000200:0.0:1788572770.113762:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572770.113767:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572770.113770:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572770.113772:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113786:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.113789:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113790:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572770.113793:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572770.113795:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572770.113797:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572770.113800:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572770.113802:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572770.113803:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572770.113805:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113807:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.113808:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113809:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572770.113811:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572770.113813:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113815:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572770.113817:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572770.113820:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572770.113822:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.113823:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572770.113825:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572770.113829:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572770.113832:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572770.113835:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572770.113836:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.113838:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572770.113840:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572770.113844:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572770.113850:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572770.113856:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.113863:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810013824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572770.113878:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572770.113881:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.113885:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810013824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572770.113902:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572770.113904:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572770.113907:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572770.113908:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572770.113909:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572770.113911:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572770.113913:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13236 00000400:00000200:0.0:1788572770.113916:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572770.113918:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572770.113920:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572770.113925:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113927:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.113931:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113934:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572770.113937:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572770.113938:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572770.113939:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572770.113943:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572770.113946:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572770.113951:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572770.113957:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113959:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.113960:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113961:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572770.113971:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572770.113977:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572770.113980:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572770.113983:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572770.113989:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572770.113995:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.113998:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572770.114003:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572770.114009:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114011:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572770.114015:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572770.114020:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572770.114026:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572770.114029:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114031:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572770.114032:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572770.114034:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572770.114040:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572770.114046:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.114053:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810013952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572770.114061:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572770.114063:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.114066:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810013952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572770.114082:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572770.114086:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572770.114093:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572770.114096:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572770.114099:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572770.114105:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572770.114114:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13236 00000400:00000200:0.0:1788572770.114121:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572770.114126:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572770.114127:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572770.114135:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114139:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114145:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114149:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572770.114155:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572770.114158:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572770.114163:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572770.114168:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572770.114173:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572770.114178:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572770.114184:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114189:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114194:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114198:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572770.114202:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572770.114206:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114209:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572770.114214:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572770.114232:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572770.114237:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114241:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572770.114246:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572770.114251:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114257:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572770.114264:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572770.114270:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572770.114277:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572770.114285:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114287:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572770.114290:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572770.114293:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572770.114296:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572770.114302:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.114311:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810014080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572770.114317:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572770.114321:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.114326:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810014080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572770.114337:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572770.114340:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572770.114346:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572770.114349:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572770.114352:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572770.114355:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572770.114361:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13236 00000400:00000200:0.0:1788572770.114368:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572770.114375:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572770.114379:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572770.114384:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114388:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114390:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114394:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572770.114399:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572770.114402:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572770.114405:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572770.114408:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572770.114411:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572770.114418:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572770.114422:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114425:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114428:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114433:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572770.114435:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572770.114440:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114443:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572770.114446:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572770.114451:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572770.114452:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114454:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572770.114456:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572770.114457:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114460:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572770.114466:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572770.114470:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572770.114473:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572770.114476:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114478:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572770.114479:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572770.114480:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572770.114484:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572770.114488:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.114495:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810014208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572770.114510:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572770.114512:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.114516:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810014208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572770.114522:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572770.114533:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572770.114538:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572770.114542:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572770.114546:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572770.114550:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572770.114554:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13236 00000400:00000200:0.0:1788572770.114562:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572770.114571:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572770.114573:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572770.114580:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114584:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114589:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114593:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572770.114598:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572770.114602:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572770.114620:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572770.114623:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572770.114634:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572770.114675:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572770.114686:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114690:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572770.114696:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114699:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572770.114702:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572770.114708:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572770.114711:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572770.114713:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572770.114716:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572770.114722:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114727:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572770.114735:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572770.114740:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114744:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572770.114746:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572770.114748:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572770.114750:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572770.114751:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.114753:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572770.114754:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572770.114757:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572770.114761:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572770.114766:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.114774:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810014336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572770.114784:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572770.114787:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.114792:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810014336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572770.114807:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572770.114814:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13246) 00000020:00000001:0.0:1788572770.497836:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572770.497839:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572770.497842:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572773.953725:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307912926, at 4307912960 00000100:00000010:0.0:1788572773.953750:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788572774.593928:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572774.593932:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572774.593936:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572775.233697:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572775.233710:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572775.233712:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572775.233716:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572775.233737:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572775.233744:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13241 00000400:00000200:0.0:1788572775.233753:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572775.233759:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572775.233763:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572775.233766:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233770:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.233772:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233775:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572775.233779:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572775.233782:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572775.233784:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572775.233788:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572775.233791:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572775.233793:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572775.233796:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233798:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.233801:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233803:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572775.233806:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572775.233810:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233813:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572775.233816:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572775.233819:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572775.233822:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.233825:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572775.233827:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572775.233835:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572775.233839:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572775.233843:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572775.233845:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.233853:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572775.233856:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572775.233860:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572775.233869:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572775.233876:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.233886:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810014464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572775.233897:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572775.233901:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.233908:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810014464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572775.233931:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572775.233935:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572775.233938:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572775.233940:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572775.233942:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572775.233945:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572775.233949:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13241 00000400:00000200:0.0:1788572775.233955:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572775.233959:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572775.233966:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572775.233972:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233978:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.233984:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.233991:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572775.234006:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572775.234010:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572775.234015:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572775.234019:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572775.234024:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572775.234029:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572775.234049:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234054:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.234063:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234069:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572775.234077:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572775.234086:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234092:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572775.234104:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572775.234113:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572775.234120:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234124:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572775.234130:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572775.234137:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234142:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572775.234150:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572775.234157:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572775.234164:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572775.234168:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234173:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572775.234177:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572775.234183:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572775.234190:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572775.234198:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.234210:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810014592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572775.234219:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572775.234223:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.234233:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810014592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572775.234249:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572775.234254:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572775.234270:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572775.234274:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572775.234289:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572775.234296:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572775.234304:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13241 00000400:00000200:0.0:1788572775.234315:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572775.234328:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572775.234334:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572775.234343:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234350:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.234359:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234367:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572775.234373:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572775.234377:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572775.234382:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572775.234387:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572775.234394:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572775.234399:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572775.234409:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234416:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.234423:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234429:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572775.234436:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572775.234445:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234453:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572775.234459:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572775.234465:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572775.234471:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234478:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572775.234485:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572775.234502:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234507:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572775.234515:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572775.234522:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572775.234530:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572775.234534:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234539:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572775.234543:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572775.234553:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572775.234562:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572775.234570:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.234581:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810014720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572775.234597:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572775.234600:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.234605:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810014720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572775.234661:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572775.234667:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572775.234673:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572775.234677:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572775.234681:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572775.234686:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572775.234693:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13241 00000400:00000200:0.0:1788572775.234700:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572775.234707:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572775.234713:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572775.234721:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234726:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.234733:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234748:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572775.234755:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572775.234758:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572775.234761:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572775.234764:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572775.234766:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572775.234771:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572775.234777:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234781:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.234787:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234792:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572775.234797:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572775.234804:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572775.234811:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572775.234815:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572775.234821:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572775.234827:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234832:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572775.234837:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572775.234845:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234850:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572775.234858:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572775.234864:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572775.234870:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572775.234874:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.234878:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572775.234882:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572775.234886:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572775.234895:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572775.234902:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.234913:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810014848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572775.234954:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572775.234960:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.234971:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810014848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572775.234993:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572775.234999:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572775.235008:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572775.235013:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572775.235018:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572775.235026:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572775.235031:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13241 00000400:00000200:0.0:1788572775.235038:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572775.235044:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572775.235050:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572775.235057:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.235064:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.235071:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.235074:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572775.235080:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572775.235084:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572775.235088:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572775.235093:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572775.235099:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572775.235104:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572775.235112:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.235118:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572775.235124:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572775.235126:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572775.235136:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572775.235140:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572775.235144:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572775.235148:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572775.235155:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572775.235160:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.235165:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572775.235168:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572775.235175:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.235179:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572775.235185:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572775.235192:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572775.235198:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572775.235201:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.235207:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572775.235211:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572775.235216:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572775.235224:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572775.235231:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.235242:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810014976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572775.235258:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572775.235263:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.235274:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810014976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572775.235301:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572775.235309:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13251) 00000020:00000001:0.0:1788572775.617911:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572775.617916:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572775.617919:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572779.073743:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307918046, at 4307918080 00000100:00000010:0.0:1788572779.073761:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000020:00000001:0.0:1788572779.713827:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572779.713832:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572779.713836:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572780.353785:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572780.353797:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572780.353800:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572780.353804:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572780.353809:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572780.353818:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13246 00000400:00000200:0.0:1788572780.353826:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572780.353832:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572780.353834:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572780.353837:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.353839:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.353841:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.353842:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572780.353846:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572780.353847:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572780.353850:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572780.353852:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572780.353854:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572780.353855:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572780.353857:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.353859:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.353860:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.353862:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572780.353864:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572780.353868:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572780.353870:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572780.353873:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572780.353891:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572780.353894:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.353897:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572780.353899:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572780.353906:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572780.353910:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572780.353914:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572780.353915:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.353919:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572780.353921:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572780.353925:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572780.353933:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572780.353940:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.353951:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810015104/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572780.353973:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572780.353977:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.353983:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810015104/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572780.354028:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572780.354032:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572780.354036:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572780.354037:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572780.354039:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572780.354042:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572780.354045:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13246 00000400:00000200:0.0:1788572780.354050:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572780.354055:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572780.354058:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572780.354062:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354064:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354073:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354078:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572780.354083:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572780.354086:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572780.354088:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572780.354093:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572780.354099:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572780.354104:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572780.354112:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354116:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354121:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354127:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572780.354134:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572780.354142:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354148:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572780.354152:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572780.354159:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572780.354164:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354167:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572780.354173:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572780.354190:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354196:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572780.354203:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572780.354209:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572780.354215:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572780.354217:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354220:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572780.354222:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572780.354226:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572780.354234:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572780.354264:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.354276:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810015232/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572780.354298:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572780.354303:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.354313:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810015232/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572780.354334:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572780.354339:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572780.354347:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572780.354349:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572780.354351:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572780.354358:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572780.354365:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13246 00000400:00000200:0.0:1788572780.354375:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572780.354382:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572780.354387:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572780.354393:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354398:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354403:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354409:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572780.354414:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572780.354418:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572780.354422:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572780.354427:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572780.354431:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572780.354434:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572780.354439:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354444:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354449:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354451:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572780.354473:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572780.354485:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354492:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572780.354497:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572780.354502:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572780.354507:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354512:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572780.354516:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572780.354523:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354528:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572780.354535:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572780.354541:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572780.354544:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572780.354547:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354552:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572780.354556:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572780.354561:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572780.354567:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572780.354573:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.354583:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810015360/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572780.354599:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572780.354604:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.354611:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810015360/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572780.354665:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572780.354668:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572780.354672:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572780.354673:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572780.354675:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572780.354677:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572780.354687:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13246 00000400:00000200:0.0:1788572780.354693:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572780.354698:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572780.354702:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572780.354709:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354715:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354720:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354725:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572780.354730:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572780.354733:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572780.354738:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572780.354742:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572780.354746:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572780.354749:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572780.354753:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354756:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354761:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354767:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572780.354773:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572780.354779:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354783:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572780.354787:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572780.354793:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572780.354798:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354803:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572780.354809:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572780.354814:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354818:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572780.354824:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572780.354829:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572780.354835:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572780.354844:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.354849:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572780.354852:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572780.354855:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572780.354863:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572780.354869:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.354881:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810015488/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572780.354894:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572780.354897:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.354904:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810015488/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572780.354917:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572780.354921:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572780.354927:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572780.354930:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572780.354933:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572780.354939:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572780.354945:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13246 00000400:00000200:0.0:1788572780.354956:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572780.354965:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572780.354971:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572780.354975:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354977:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.354982:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.354985:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572780.354988:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572780.354991:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572780.354994:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572780.354998:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572780.355001:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572780.355014:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572780.355023:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.355029:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572780.355035:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572780.355040:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572780.355045:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572780.355053:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572780.355058:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572780.355063:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572780.355069:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572780.355075:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.355080:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572780.355084:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572780.355090:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.355092:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572780.355096:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572780.355103:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572780.355110:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572780.355115:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.355120:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572780.355124:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572780.355127:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572780.355135:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572780.355143:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.355154:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810015616/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572780.355173:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572780.355177:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.355186:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810015616/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572780.355204:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572780.355220:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13257) 00000020:00000001:0.0:1788572780.737836:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572780.737842:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572780.737845:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572784.193769:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307923166, at 4307923200 00000100:00000010:0.0:1788572784.193789:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788572784.833800:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572784.833804:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572784.833807:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572785.473752:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572785.473764:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572785.473767:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572785.473771:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572785.473776:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572785.473783:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13252 00000400:00000200:0.0:1788572785.473794:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572785.473801:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572785.473804:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572785.473808:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.473811:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.473813:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.473816:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572785.473820:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572785.473822:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572785.473825:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572785.473829:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572785.473831:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572785.473833:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572785.473836:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.473837:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.473839:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.473859:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572785.473862:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572785.473865:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572785.473868:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572785.473871:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572785.473874:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572785.473877:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.473879:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572785.473881:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572785.473888:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572785.473892:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572785.473896:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572785.473898:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.473901:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572785.473904:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572785.473908:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572785.473916:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572785.473925:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.473934:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810015744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572785.473944:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572785.473948:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.473954:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810015744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572785.473978:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572785.473983:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572785.473988:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572785.473990:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572785.473993:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572785.473996:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572785.474000:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13252 00000400:00000200:0.0:1788572785.474016:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572785.474028:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572785.474035:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572785.474043:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474046:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474053:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474058:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572785.474066:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572785.474071:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572785.474076:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572785.474081:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572785.474086:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572785.474092:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572785.474099:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474101:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474103:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474105:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572785.474108:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572785.474111:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474114:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572785.474116:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572785.474123:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572785.474125:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474127:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572785.474132:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572785.474134:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474136:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572785.474139:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572785.474142:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572785.474145:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572785.474159:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474164:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572785.474169:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572785.474174:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572785.474183:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572785.474190:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.474198:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810015872/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572785.474207:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572785.474210:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.474214:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810015872/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572785.474224:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572785.474226:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572785.474229:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572785.474230:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572785.474232:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572785.474233:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572785.474235:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13252 00000400:00000200:0.0:1788572785.474239:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572785.474242:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572785.474245:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572785.474248:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474250:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474252:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474255:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572785.474257:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572785.474259:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572785.474261:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572785.474263:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572785.474266:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572785.474274:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572785.474279:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474281:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474283:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474286:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572785.474289:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572785.474292:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474294:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572785.474297:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572785.474300:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572785.474303:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474306:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572785.474308:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572785.474314:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474316:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572785.474321:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572785.474328:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572785.474334:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572785.474337:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474344:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572785.474347:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572785.474353:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572785.474363:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572785.474369:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.474376:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810016000/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572785.474386:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572785.474389:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.474394:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810016000/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572785.474403:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572785.474410:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572785.474414:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572785.474415:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572785.474417:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572785.474420:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572785.474423:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13252 00000400:00000200:0.0:1788572785.474430:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572785.474437:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572785.474440:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572785.474446:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474448:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474450:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474451:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572785.474455:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572785.474457:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572785.474460:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572785.474462:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572785.474466:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572785.474472:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572785.474480:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474487:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474495:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474502:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572785.474510:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572785.474519:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474525:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572785.474529:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572785.474533:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572785.474536:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474541:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572785.474551:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572785.474557:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474559:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572785.474564:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572785.474580:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572785.474588:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572785.474592:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474598:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572785.474603:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572785.474608:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572785.474615:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788572785.474621:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.474680:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810016128/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572785.474694:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572785.474700:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.474711:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810016128/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572785.474728:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572785.474731:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572785.474736:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572785.474739:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572785.474743:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572785.474746:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572785.474751:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13252 00000400:00000200:0.0:1788572785.474757:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572785.474761:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572785.474763:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572785.474767:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474770:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474775:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474786:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572785.474790:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572785.474793:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572785.474796:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572785.474800:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572785.474804:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572785.474810:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572785.474816:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474817:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572785.474820:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474825:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572785.474831:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572785.474837:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572785.474840:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572785.474842:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572785.474848:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572785.474851:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474855:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572785.474859:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572785.474864:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474868:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572785.474873:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572785.474877:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572785.474883:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572785.474886:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572785.474889:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572785.474892:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572785.474895:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572785.474902:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572785.474908:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.474916:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810016256/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572785.474966:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572785.474970:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.474978:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810016256/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572785.474993:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572785.474998:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13262) 00000020:00000001:0.0:1788572785.857848:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572785.857852:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572785.857855:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572786.626338:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572786.626343:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572786.626345:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572786.626348:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572786.626352:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572786.626356:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13257 00000400:00000200:0.0:1788572786.626363:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572786.626369:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572786.626372:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572786.626374:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626377:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626379:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626381:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572786.626384:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572786.626385:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572786.626386:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572786.626388:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572786.626390:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572786.626391:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572786.626394:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626404:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626406:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626409:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572786.626413:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572786.626417:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626420:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572786.626423:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572786.626427:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572786.626430:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626433:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572786.626435:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572786.626442:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572786.626445:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572786.626449:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572786.626450:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626452:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572786.626454:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572786.626456:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572786.626460:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572786.626463:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.626468:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810016512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572786.626475:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572786.626477:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.626481:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810016512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572786.626499:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572786.626502:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572786.626506:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572786.626508:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572786.626509:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572786.626513:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572786.626525:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13257 00000400:00000200:0.0:1788572786.626532:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572786.626536:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572786.626541:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572786.626547:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626550:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626557:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626561:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572786.626564:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572786.626567:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572786.626571:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572786.626575:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572786.626581:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572786.626585:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572786.626592:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626597:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626602:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626606:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572786.626613:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572786.626620:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626672:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572786.626676:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572786.626683:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572786.626686:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626690:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572786.626697:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572786.626704:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626709:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572786.626717:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572786.626723:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572786.626729:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572786.626738:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626742:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572786.626745:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572786.626748:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572786.626758:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572786.626766:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.626776:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810016640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572786.626791:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572786.626795:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.626803:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810016640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572786.626817:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572786.626822:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572786.626828:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572786.626832:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572786.626834:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572786.626841:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572786.626848:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13257 00000400:00000200:0.0:1788572786.626858:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572786.626865:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572786.626870:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572786.626878:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626882:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626889:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626894:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572786.626899:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572786.626902:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572786.626907:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572786.626910:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572786.626914:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572786.626928:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572786.626936:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626942:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.626947:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626952:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572786.626959:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572786.626966:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572786.626971:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572786.626977:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572786.626982:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572786.626989:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.626995:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572786.627000:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572786.627006:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627012:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572786.627017:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572786.627023:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572786.627030:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572786.627033:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627036:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572786.627039:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572786.627043:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572786.627050:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572786.627058:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.627068:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810016768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572786.627092:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572786.627096:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.627104:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810016768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572786.627122:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572786.627134:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572786.627144:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572786.627148:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572786.627151:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572786.627157:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572786.627163:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13257 00000400:00000200:0.0:1788572786.627173:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572786.627180:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572786.627187:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572786.627194:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627198:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.627204:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627219:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572786.627225:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572786.627229:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572786.627232:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572786.627236:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572786.627240:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572786.627243:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572786.627247:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627253:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.627258:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627263:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572786.627269:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572786.627275:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627280:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572786.627283:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572786.627288:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572786.627292:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627295:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572786.627306:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572786.627314:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627319:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572786.627326:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572786.627331:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572786.627338:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572786.627341:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627345:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572786.627349:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572786.627353:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572786.627361:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572786.627367:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.627378:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810016896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572786.627404:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572786.627410:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.627419:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810016896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572786.627438:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572786.627444:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572786.627451:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572786.627455:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572786.627459:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572786.627465:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572786.627472:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13257 00000400:00000200:0.0:1788572786.627482:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572786.627490:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572786.627495:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572786.627502:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627506:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.627510:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627520:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572786.627526:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572786.627529:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572786.627534:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572786.627537:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572786.627541:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572786.627546:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572786.627552:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627556:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572786.627561:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627566:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572786.627571:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572786.627578:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572786.627584:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572786.627590:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572786.627595:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572786.627601:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627605:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572786.627610:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572786.627616:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627619:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572786.627623:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572786.627668:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572786.627674:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572786.627678:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.627684:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572786.627688:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572786.627693:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572786.627701:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572786.627709:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.627719:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810017024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572786.627758:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572786.627764:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.627775:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810017024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572786.627801:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572786.627807:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13263) 00000020:00000001:0.0:1788572789.953864:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572789.953868:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572789.953871:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572790.465734:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307929439, at 4307929472 00000100:00000010:0.0:1788572790.465753:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000020:00000001:0.0:1788572790.977821:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572790.977826:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572790.977831:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572792.129799:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572792.129811:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572792.129813:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572792.129817:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572792.129822:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572792.129831:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13258 00000400:00000200:0.0:1788572792.129843:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572792.129852:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572792.129856:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572792.129859:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.129863:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.129865:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.129867:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572792.129871:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572792.129873:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572792.129876:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572792.129904:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572792.129906:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572792.129908:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572792.129911:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.129912:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.129914:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.129915:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572792.129917:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572792.129920:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572792.129922:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572792.129924:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572792.129928:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572792.129930:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.129932:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572792.129934:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572792.129940:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572792.129943:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572792.129947:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572792.129949:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.129952:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572792.129955:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572792.129959:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572792.129967:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572792.129975:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.129983:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810017152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572792.129994:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572792.130000:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.130006:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810017152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572792.130032:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572792.130036:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572792.130048:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572792.130050:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572792.130052:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572792.130056:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572792.130060:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13258 00000400:00000200:0.0:1788572792.130066:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572792.130071:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572792.130079:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572792.130090:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130099:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130109:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130118:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572792.130128:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572792.130131:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572792.130134:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572792.130139:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572792.130142:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572792.130148:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572792.130158:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130162:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130166:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130173:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572792.130178:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572792.130186:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130190:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572792.130202:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572792.130217:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572792.130221:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130232:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572792.130243:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572792.130260:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130264:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572792.130277:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572792.130287:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572792.130291:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572792.130292:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130294:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572792.130296:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572792.130298:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572792.130308:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572792.130316:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.130326:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810017280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572792.130337:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572792.130341:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.130349:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810017280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572792.130379:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572792.130389:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572792.130398:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572792.130403:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572792.130407:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572792.130419:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572792.130428:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13258 00000400:00000200:0.0:1788572792.130444:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572792.130463:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572792.130470:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572792.130474:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130475:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130477:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130487:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572792.130492:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572792.130496:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572792.130502:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572792.130506:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572792.130514:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572792.130519:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572792.130524:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130529:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130533:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130536:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572792.130543:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572792.130545:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130547:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572792.130550:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572792.130554:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572792.130558:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130561:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572792.130566:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572792.130577:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130580:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572792.130597:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572792.130604:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572792.130616:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572792.130620:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130626:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572792.130631:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572792.130686:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572792.130691:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572792.130695:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.130705:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810017408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572792.130745:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572792.130752:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.130766:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810017408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572792.130776:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572792.130781:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572792.130794:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572792.130801:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572792.130805:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572792.130811:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572792.130814:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13258 00000400:00000200:0.0:1788572792.130818:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572792.130823:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572792.130831:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572792.130839:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130846:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130854:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130861:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572792.130871:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572792.130881:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572792.130884:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572792.130887:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572792.130895:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572792.130900:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572792.130913:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130927:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.130933:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130937:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572792.130949:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572792.130959:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572792.130967:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572792.130970:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572792.130977:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572792.130983:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.130988:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572792.130993:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572792.130999:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.131006:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572792.131013:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572792.131019:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572792.131025:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572792.131030:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.131035:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572792.131041:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572792.131045:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572792.131052:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572792.131064:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.131075:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810017536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572792.131108:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572792.131113:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.131126:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810017536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572792.131146:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572792.131150:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572792.131154:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572792.131157:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572792.131161:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572792.131165:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572792.131176:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13258 00000400:00000200:0.0:1788572792.131182:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572792.131187:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572792.131195:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572792.131209:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.131215:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.131225:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.131233:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572792.131243:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572792.131247:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572792.131254:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572792.131260:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572792.131272:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572792.131281:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572792.131300:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.131307:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572792.131310:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572792.131314:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572792.131318:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572792.131331:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572792.131342:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572792.131347:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572792.131355:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572792.131363:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.131368:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572792.131371:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572792.131375:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.131378:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572792.131388:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572792.131397:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572792.131405:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572792.131418:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572792.131426:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572792.131435:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572792.131445:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572792.131456:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572792.131461:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.131470:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810017664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572792.131488:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572792.131493:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.131500:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810017664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572792.131526:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572792.131538:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13268) 00000020:00000001:0.0:1788572795.073860:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572795.073865:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572795.073869:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572795.969740:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307934943, at 4307934976 00000100:00000010:0.0:1788572795.969755:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000020:00000001:0.0:1788572796.097847:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572796.097852:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572796.097855:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572799.233768:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572799.233773:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572799.233777:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572799.233924:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572799.233939:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572799.233945:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572799.233948:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572800.193893:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572800.193896:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572800.193899:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572801.089756:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307940062, at 4307940096 00000100:00000010:0.0:1788572801.089786:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788572801.217855:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572801.217859:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572801.217862:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572805.313819:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572805.313823:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572805.313827:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572806.209748:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307945182, at 4307945216 00000100:00000010:0.0:1788572806.209767:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788572806.337771:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572806.337775:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572806.337777:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572810.433875:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572810.433879:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572810.433882:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572811.457891:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572811.457895:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572811.457898:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572812.481817:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307951455, at 4307951488 00000100:00000010:0.0:1788572812.481839:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000020:00000001:0.0:1788572815.553880:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572815.553883:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572815.553886:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572816.577818:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572816.577823:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572816.577827:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572817.985750:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307956958, at 4307956992 00000100:00000010:0.0:1788572817.985766:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788572820.673796:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572820.673799:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572820.673801:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572821.697890:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572821.697894:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572821.697897:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572823.105724:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307962078, at 4307962112 00000100:00000010:0.0:1788572823.105741:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788572825.793948:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572825.793954:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572825.793957:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572826.817803:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572826.817807:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572826.817810:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572828.225716:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307967198, at 4307967232 00000100:00000010:0.0:1788572828.225729:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00080000:0.0:1788572829.505772:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572829.505784:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572829.505787:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572829.505791:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572829.505796:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572829.505803:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13296 00000400:00000200:0.0:1788572829.505812:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572829.505817:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572829.505821:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572829.505823:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.505826:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.505828:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.505830:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572829.505833:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572829.505834:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572829.505836:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572829.505839:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572829.505841:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572829.505843:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572829.505847:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.505849:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.505866:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.505869:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572829.505872:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572829.505876:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572829.505879:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572829.505883:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572829.505886:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572829.505889:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.505891:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572829.505894:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572829.505899:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572829.505903:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572829.505906:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572829.505907:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.505909:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572829.505912:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572829.505916:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572829.505921:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572829.505927:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.505935:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810022400/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572829.505944:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572829.505948:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.505951:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810022400/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572829.505971:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572829.505975:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572829.505979:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572829.505980:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572829.505982:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572829.505984:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572829.505994:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13296 00000400:00000200:0.0:1788572829.505999:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572829.506002:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572829.506005:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572829.506024:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506026:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506028:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506031:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572829.506033:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572829.506035:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572829.506036:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572829.506039:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572829.506041:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572829.506044:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572829.506047:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506049:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506051:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506053:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572829.506056:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572829.506059:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506062:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572829.506063:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572829.506069:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572829.506071:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506073:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572829.506077:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572829.506080:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506082:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572829.506085:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572829.506088:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572829.506091:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572829.506096:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506098:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572829.506100:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572829.506102:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572829.506106:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572829.506109:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.506114:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810022528/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572829.506122:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572829.506125:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.506129:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810022528/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572829.506138:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572829.506141:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572829.506145:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572829.506146:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572829.506148:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572829.506150:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572829.506153:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13296 00000400:00000200:0.0:1788572829.506157:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572829.506160:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572829.506162:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572829.506165:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506167:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506169:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506171:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572829.506173:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572829.506175:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572829.506176:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572829.506178:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572829.506180:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572829.506187:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572829.506191:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506193:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506195:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506197:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572829.506199:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572829.506202:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506205:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572829.506207:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572829.506210:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572829.506213:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506215:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572829.506217:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572829.506220:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506222:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572829.506225:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572829.506228:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572829.506230:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572829.506232:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506234:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572829.506235:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572829.506237:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572829.506241:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572829.506245:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.506249:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810022656/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572829.506257:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572829.506259:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.506262:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810022656/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572829.506271:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572829.506277:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572829.506280:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572829.506282:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572829.506284:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572829.506287:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572829.506289:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13296 00000400:00000200:0.0:1788572829.506293:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572829.506296:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572829.506299:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572829.506302:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506304:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506306:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506308:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572829.506310:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572829.506312:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572829.506313:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572829.506315:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572829.506318:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572829.506320:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572829.506323:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506325:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506326:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506328:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572829.506331:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572829.506333:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506336:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572829.506338:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572829.506341:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572829.506343:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506345:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572829.506351:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572829.506354:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506356:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572829.506359:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572829.506361:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572829.506364:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572829.506365:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506367:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572829.506368:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572829.506370:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572829.506374:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572829.506377:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.506382:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810022784/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572829.506390:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572829.506392:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.506396:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810022784/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572829.506405:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572829.506407:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572829.506411:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572829.506413:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572829.506414:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572829.506417:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572829.506419:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13296 00000400:00000200:0.0:1788572829.506424:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572829.506427:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572829.506430:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572829.506433:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506435:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506437:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506443:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572829.506446:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572829.506447:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572829.506449:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572829.506451:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572829.506453:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572829.506455:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572829.506458:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506460:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572829.506463:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506464:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572829.506467:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572829.506470:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572829.506473:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572829.506475:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572829.506477:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572829.506480:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506482:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572829.506484:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572829.506486:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506488:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572829.506491:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572829.506493:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572829.506496:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572829.506497:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.506499:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572829.506501:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572829.506503:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572829.506506:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572829.506510:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.506514:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810022912/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572829.506531:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572829.506533:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.506537:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810022912/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572829.506545:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.506550:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13306) 00000100:00080000:0.0:1788572830.274406:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572830.274411:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572830.274413:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572830.274416:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572830.274419:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572830.274423:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572830.274429:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572830.274435:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572830.274437:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572830.274440:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572830.274442:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572830.274444:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572830.274445:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572830.274447:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572830.274449:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572830.274450:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572830.274452:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572830.274453:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572830.274455:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572830.274457:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572830.274459:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572830.274460:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572830.274461:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572830.274471:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572830.274481:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572830.274483:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572830.274485:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572830.274488:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572830.274490:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572830.274491:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572830.274493:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572830.274496:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572830.274498:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572830.274500:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572830.274502:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572830.274503:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572830.274504:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572830.274507:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572830.274511:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572830.274514:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572830.274520:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810023168/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572830.274530:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572830.274533:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572830.274538:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810023168/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572830.274560:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572830.274567:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13306) 00000020:00000001:0.0:1788572830.913770:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572830.913774:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572830.913776:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572831.937795:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572831.937799:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572831.937802:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572834.113735:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307973087, at 4307973120 00000100:00000010:0.0:1788572834.113752:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000100:00080000:0.0:1788572835.649774:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572835.649787:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572835.649791:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572835.649803:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572835.649810:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572835.649822:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572835.649834:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572835.649845:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572835.649850:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572835.649854:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.649858:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.649861:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.649863:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572835.649867:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572835.649871:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572835.649875:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572835.649879:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572835.649881:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572835.649884:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572835.649890:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.649892:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.649894:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.649898:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572835.649902:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572835.649906:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572835.649911:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572835.649914:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572835.649918:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572835.649920:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.649922:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572835.649937:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572835.649944:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572835.649949:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572835.649954:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572835.649956:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.649960:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572835.649964:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572835.649970:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572835.649981:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572835.649993:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.650012:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810023296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572835.650028:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572835.650035:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.650041:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810023296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572835.650083:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572835.650090:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572835.650095:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572835.650098:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572835.650100:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572835.650104:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572835.650109:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572835.650116:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572835.650123:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572835.650127:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572835.650150:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650152:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.650154:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650156:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572835.650158:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572835.650167:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572835.650170:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572835.650173:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572835.650175:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572835.650179:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572835.650185:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650189:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.650196:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650199:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572835.650203:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572835.650223:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650225:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572835.650227:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572835.650236:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572835.650239:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650242:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572835.650247:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572835.650251:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650254:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572835.650261:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572835.650266:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572835.650274:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572835.650279:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650283:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572835.650285:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572835.650290:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572835.650296:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572835.650301:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.650312:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810023424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572835.650354:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572835.650361:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.650376:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810023424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572835.650394:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572835.650409:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572835.650422:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572835.650425:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572835.650437:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572835.650448:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572835.650454:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572835.650472:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572835.650492:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572835.650500:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572835.650512:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650522:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.650533:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650538:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572835.650554:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572835.650557:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572835.650560:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572835.650565:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572835.650572:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572835.650575:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572835.650583:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650587:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.650597:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650600:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572835.650603:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572835.650611:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650616:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572835.650626:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572835.650631:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572835.650678:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650682:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572835.650685:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572835.650688:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650690:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572835.650696:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572835.650699:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572835.650704:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572835.650707:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.650709:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572835.650711:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572835.650714:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572835.650719:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572835.650732:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.650754:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810023552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572835.650770:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572835.650778:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.650793:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810023552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572835.650822:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572835.650836:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572835.650841:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572835.650844:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572835.650847:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572835.650866:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572835.650872:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572835.650931:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572835.650941:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572835.650945:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572835.650976:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.650992:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.651007:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651022:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572835.651040:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572835.651046:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572835.651059:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572835.651069:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572835.651083:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572835.651099:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572835.651125:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651148:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.651151:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651155:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572835.651161:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572835.651165:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651173:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572835.651182:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572835.651192:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572835.651204:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651213:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572835.651218:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572835.651229:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651239:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572835.651251:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572835.651257:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572835.651262:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572835.651264:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651265:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572835.651267:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572835.651269:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572835.651278:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572835.651292:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.651306:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810023680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572835.651323:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572835.651326:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.651331:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810023680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572835.651345:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572835.651349:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572835.651355:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572835.651357:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572835.651359:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572835.651366:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572835.651371:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13301 00000400:00000200:0.0:1788572835.651394:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572835.651405:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572835.651410:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572835.651426:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651431:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.651442:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651453:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572835.651466:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572835.651472:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572835.651482:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572835.651490:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572835.651495:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572835.651504:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572835.651516:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651525:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572835.651541:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651548:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572835.651556:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572835.651566:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572835.651582:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572835.651595:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572835.651605:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572835.651612:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651615:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572835.651619:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572835.651625:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651666:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572835.651672:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572835.651676:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572835.651678:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572835.651681:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572835.651684:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572835.651687:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572835.651690:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572835.651695:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572835.651705:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.651727:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810023808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572835.651757:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572835.651770:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.651796:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810023808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572835.651809:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572835.651815:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13312) 00000020:00000001:0.0:1788572836.033925:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572836.033934:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572836.033940:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572837.057892:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572837.057897:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572837.057900:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572837.121724:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572837.121731:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572837.121735:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572837.121876:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572837.121904:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572837.121909:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572837.121913:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572839.489737:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307978463, at 4307978496 00000100:00000010:0.0:1788572839.489757:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788572841.153872:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572841.153876:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572841.153879:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572842.177802:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572842.177805:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572842.177808:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572844.609758:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307983582, at 4307983616 00000100:00000010:0.0:1788572844.609772:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788572846.273813:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572846.273816:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572846.273818:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572847.297848:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572847.297852:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572847.297855:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572849.729709:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307988703, at 4307988736 00000100:00000010:0.0:1788572849.729720:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000020:00000001:0.0:1788572851.393806:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572851.393810:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572851.393812:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572852.417875:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572852.417879:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572852.417882:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572855.617754:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307994591, at 4307994624 00000100:00000010:0.0:1788572855.617769:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788572856.513881:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572856.513888:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572856.513892:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572857.537868:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572857.537871:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572857.537874:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572860.993780:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4307999966, at 4308000000 00000100:00000010:0.0:1788572860.993800:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788572861.633820:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572861.633824:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572861.633827:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572862.657910:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572862.657916:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572862.657921:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572866.113784:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308005087, at 4308005120 00000100:00000010:0.0:1788572866.113795:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788572866.753792:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572866.753796:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572866.753799:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572867.777792:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572867.777795:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572867.777798:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572871.233710:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308010206, at 4308010240 00000100:00000010:0.0:1788572871.233724:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788572871.873783:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572871.873786:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572871.873788:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572872.513789:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572872.513800:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572872.513802:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572872.513807:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572872.513827:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572872.513834:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13339 00000400:00000200:0.0:1788572872.513843:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572872.513849:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572872.513853:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572872.513857:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.513860:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.513877:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.513880:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572872.513885:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572872.513886:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572872.513889:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572872.513893:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572872.513896:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572872.513898:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572872.513902:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.513904:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.513906:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.513908:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572872.513911:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572872.513914:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572872.513917:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572872.513920:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572872.513923:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572872.513926:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.513929:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572872.513931:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572872.513937:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572872.513940:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572872.513945:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572872.513946:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.513953:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572872.513957:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572872.513961:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572872.513968:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572872.513975:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.513984:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810028544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572872.513996:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572872.514000:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.514005:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810028544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572872.514029:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572872.514033:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572872.514036:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572872.514038:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572872.514040:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572872.514043:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572872.514045:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13339 00000400:00000200:0.0:1788572872.514050:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572872.514053:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572872.514056:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572872.514060:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514063:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514066:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514069:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572872.514073:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572872.514075:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572872.514079:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572872.514082:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572872.514087:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572872.514091:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572872.514104:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514110:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514115:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514119:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572872.514125:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572872.514129:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514133:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572872.514135:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572872.514142:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572872.514144:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514148:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572872.514154:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572872.514158:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514162:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572872.514168:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572872.514173:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572872.514178:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572872.514180:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514183:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572872.514186:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572872.514188:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572872.514194:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788572872.514201:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.514209:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810028672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572872.514229:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572872.514234:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.514242:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810028672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572872.514259:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572872.514263:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572872.514279:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572872.514282:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572872.514286:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572872.514293:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572872.514299:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13339 00000400:00000200:0.0:1788572872.514314:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572872.514321:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572872.514326:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572872.514334:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514339:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514344:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514347:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572872.514349:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572872.514352:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572872.514355:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572872.514367:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572872.514372:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572872.514376:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572872.514381:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514387:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514392:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514396:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572872.514401:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572872.514406:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514411:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572872.514413:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572872.514417:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572872.514422:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514427:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572872.514431:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572872.514443:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514448:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572872.514453:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572872.514459:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572872.514463:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572872.514466:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514471:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572872.514475:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572872.514479:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572872.514486:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572872.514493:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.514501:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810028800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572872.514515:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572872.514519:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.514526:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810028800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572872.514545:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572872.514550:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572872.514558:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572872.514561:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572872.514564:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572872.514569:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572872.514576:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13339 00000400:00000200:0.0:1788572872.514582:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572872.514588:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572872.514591:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572872.514598:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514604:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514609:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514620:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572872.514625:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572872.514628:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572872.514632:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572872.514665:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572872.514670:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572872.514676:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572872.514684:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514690:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514696:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514702:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572872.514708:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572872.514715:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514721:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572872.514725:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572872.514730:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572872.514735:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514739:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572872.514743:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572872.514749:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514753:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572872.514758:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572872.514764:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572872.514769:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572872.514772:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.514776:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572872.514779:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572872.514783:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572872.514788:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572872.514793:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.514804:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810028928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572872.514838:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572872.514843:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.514851:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810028928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572872.514883:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572872.514888:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572872.514898:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572872.514902:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572872.514906:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572872.514913:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572872.514918:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13339 00000400:00000200:0.0:1788572872.514928:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572872.514934:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572872.514941:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572872.514947:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514952:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514957:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514960:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572872.514963:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572872.514966:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572872.514969:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572872.514973:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572872.514976:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572872.514981:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572872.514987:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.514991:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572872.514997:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572872.515001:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572872.515013:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572872.515021:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572872.515027:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572872.515030:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572872.515035:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572872.515041:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.515046:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572872.515050:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572872.515055:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.515060:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572872.515066:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572872.515071:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572872.515077:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572872.515080:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572872.515085:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572872.515087:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572872.515091:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572872.515097:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572872.515104:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.515115:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810029056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572872.515133:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572872.515138:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.515147:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810029056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572872.515159:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572872.515166:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13349) 00000020:00000001:0.0:1788572872.897854:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572872.897860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572872.897864:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572873.986327:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572873.986338:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572873.986356:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572873.986361:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572873.986367:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572873.986373:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13344 00000400:00000200:0.0:1788572873.986385:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572873.986393:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572873.986397:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572873.986401:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986405:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.986408:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986412:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572873.986416:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572873.986419:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572873.986422:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572873.986425:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572873.986428:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572873.986431:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572873.986435:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986437:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.986439:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986442:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572873.986446:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572873.986449:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986453:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572873.986457:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572873.986460:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572873.986464:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.986467:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572873.986470:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572873.986477:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572873.986480:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572873.986491:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572873.986502:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.986506:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572873.986509:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572873.986513:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572873.986521:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572873.986527:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.986537:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810029312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572873.986549:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572873.986553:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.986559:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810029312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572873.986593:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572873.986597:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572873.986601:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572873.986603:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572873.986605:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572873.986609:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572873.986612:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13344 00000400:00000200:0.0:1788572873.986617:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572873.986621:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572873.986624:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572873.986628:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986631:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.986666:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986671:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572873.986677:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572873.986681:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572873.986685:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572873.986691:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572873.986704:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572873.986710:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572873.986716:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986720:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.986726:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986733:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572873.986738:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572873.986743:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986749:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572873.986753:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572873.986761:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572873.986767:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.986771:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572873.986779:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572873.986783:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.986787:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572873.986792:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572873.986795:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572873.986802:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572873.986804:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.986807:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572873.986809:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572873.986812:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572873.986818:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572873.986824:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.986834:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810029440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572873.986846:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572873.986851:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.986859:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810029440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572873.986906:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572873.986912:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572873.986920:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572873.986924:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572873.986929:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572873.986941:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572873.986950:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13344 00000400:00000200:0.0:1788572873.986973:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572873.986981:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572873.986984:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572873.986988:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986990:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.986994:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.986999:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572873.987001:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572873.987004:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572873.987007:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572873.987011:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572873.987016:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572873.987018:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572873.987025:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987028:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.987031:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987034:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572873.987036:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572873.987039:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987042:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572873.987051:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572873.987055:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572873.987059:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987062:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572873.987066:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572873.987071:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987075:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572873.987089:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572873.987093:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572873.987098:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572873.987100:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987103:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572873.987105:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572873.987109:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572873.987113:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572873.987119:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.987126:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810029568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572873.987137:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572873.987140:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.987145:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810029568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572873.987152:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572873.987158:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572873.987163:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572873.987166:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572873.987167:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572873.987173:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572873.987177:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13344 00000400:00000200:0.0:1788572873.987181:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572873.987183:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572873.987187:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572873.987199:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987204:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.987208:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987211:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572873.987213:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572873.987217:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572873.987219:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572873.987222:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572873.987225:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572873.987229:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572873.987233:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987236:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.987240:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987244:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572873.987248:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572873.987253:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987256:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572873.987257:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572873.987261:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572873.987265:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987268:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572873.987272:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572873.987277:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987279:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572873.987284:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572873.987288:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572873.987291:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572873.987292:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987294:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572873.987295:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572873.987296:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572873.987299:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572873.987306:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.987314:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810029696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572873.987332:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572873.987334:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.987342:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810029696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572873.987359:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572873.987364:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572873.987370:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572873.987373:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572873.987376:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572873.987382:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572873.987386:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13344 00000400:00000200:0.0:1788572873.987391:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572873.987395:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572873.987401:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572873.987404:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987406:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.987408:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987411:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572873.987417:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572873.987420:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572873.987425:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572873.987427:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572873.987433:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572873.987439:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572873.987448:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987453:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572873.987464:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987468:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572873.987471:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572873.987475:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572873.987479:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572873.987484:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572873.987489:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572873.987492:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987496:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572873.987499:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572873.987502:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987503:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572873.987509:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572873.987513:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572873.987518:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572873.987519:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.987524:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572873.987526:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572873.987530:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572873.987538:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572873.987541:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.987545:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810029824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572873.987557:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572873.987561:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.987568:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810029824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572873.987592:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572873.987598:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13350) 00000800:00000200:0.0:1788572875.009776:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572875.009783:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572875.009787:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572875.010032:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572875.010059:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572875.010072:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572875.010076:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572876.993810:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572876.993813:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572876.993816:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572877.825781:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308016799, at 4308016832 00000100:00000010:0.0:1788572877.825802:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788572878.017998:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572878.018008:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572878.018011:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572879.169802:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572879.169817:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572879.169820:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572879.169826:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572879.169831:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572879.169841:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13345 00000400:00000200:0.0:1788572879.169853:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572879.169861:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572879.169866:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572879.169870:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.169874:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.169877:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.169880:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572879.169886:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572879.169889:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572879.169893:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572879.169897:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572879.169900:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572879.169903:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572879.169907:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.169934:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.169937:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.169939:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572879.169943:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572879.169948:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572879.169951:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572879.169955:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572879.169959:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572879.169963:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.169998:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572879.170002:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572879.170011:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572879.170016:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572879.170021:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572879.170023:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170027:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572879.170031:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572879.170035:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572879.170045:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572879.170054:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.170066:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810029952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572879.170079:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572879.170085:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.170091:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810029952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572879.170120:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572879.170124:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572879.170128:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572879.170130:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572879.170132:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572879.170135:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572879.170145:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13345 00000400:00000200:0.0:1788572879.170151:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572879.170155:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572879.170158:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572879.170162:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170165:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.170167:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170170:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572879.170174:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572879.170176:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572879.170178:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572879.170181:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572879.170183:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572879.170186:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572879.170190:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170192:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.170195:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170198:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572879.170205:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572879.170214:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170223:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572879.170228:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572879.170237:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572879.170248:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170256:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572879.170275:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572879.170283:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170289:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572879.170299:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572879.170305:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572879.170313:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572879.170329:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170334:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572879.170342:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572879.170349:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572879.170361:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572879.170369:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.170377:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810030080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572879.170389:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572879.170392:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.170401:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810030080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572879.170443:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572879.170455:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572879.170460:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572879.170465:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572879.170469:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572879.170478:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572879.170491:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13345 00000400:00000200:0.0:1788572879.170523:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572879.170533:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572879.170538:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572879.170550:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170558:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.170566:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170574:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572879.170587:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572879.170596:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572879.170601:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572879.170607:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572879.170618:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572879.170632:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572879.170672:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170675:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.170677:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170680:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572879.170683:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572879.170694:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572879.170703:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572879.170712:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572879.170719:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572879.170725:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170730:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572879.170735:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572879.170747:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170756:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572879.170764:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572879.170769:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572879.170774:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572879.170778:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.170784:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572879.170788:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572879.170793:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572879.170802:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572879.170809:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.170820:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810030208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572879.170859:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572879.170869:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.170889:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810030208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572879.170933:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572879.170954:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572879.170983:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572879.170993:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572879.170997:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572879.171011:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572879.171025:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13345 00000400:00000200:0.0:1788572879.171034:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572879.171047:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572879.171056:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572879.171071:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171080:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.171092:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171099:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572879.171102:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572879.171103:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572879.171106:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572879.171111:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572879.171117:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572879.171123:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572879.171136:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171148:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.171158:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171169:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572879.171174:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572879.171180:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171186:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572879.171191:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572879.171199:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572879.171211:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171218:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572879.171231:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572879.171240:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171245:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572879.171257:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572879.171271:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572879.171284:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572879.171293:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171304:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572879.171310:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572879.171316:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572879.171326:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572879.171334:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.171351:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810030336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572879.171392:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572879.171399:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.171414:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810030336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572879.171451:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572879.171459:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572879.171473:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572879.171482:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572879.171487:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572879.171499:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572879.171514:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13345 00000400:00000200:0.0:1788572879.171556:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572879.171568:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572879.171576:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572879.171589:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171598:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.171609:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171625:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572879.171658:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572879.171662:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572879.171668:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572879.171672:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572879.171682:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572879.171689:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572879.171707:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171711:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572879.171718:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171725:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572879.171735:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572879.171740:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572879.171749:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572879.171753:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572879.171760:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572879.171772:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171783:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572879.171790:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572879.171799:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171802:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572879.171816:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572879.171819:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572879.171825:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572879.171829:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572879.171834:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572879.171839:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572879.171843:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572879.171850:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572879.171858:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.171868:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810030464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572879.171936:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572879.171943:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.171957:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810030464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572879.171990:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572879.172003:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13355) 00000020:00000001:0.0:1788572882.113808:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572882.113811:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572882.113814:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572883.009749:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308021983, at 4308022016 00000100:00000010:0.0:1788572883.009762:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788572883.140662:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572883.140666:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572883.140669:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572884.289740:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572884.289751:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572884.289754:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572884.289757:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572884.289761:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572884.289768:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13350 00000400:00000200:0.0:1788572884.289777:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572884.289783:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572884.289786:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572884.289790:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.289794:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.289796:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.289798:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572884.289803:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572884.289805:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572884.289807:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572884.289826:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572884.289829:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572884.289832:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572884.289836:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.289838:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.289840:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.289842:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572884.289845:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572884.289849:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572884.289852:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572884.289854:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572884.289858:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572884.289860:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.289863:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572884.289865:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572884.289871:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572884.289875:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572884.289879:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572884.289881:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.289884:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572884.289887:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572884.289891:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572884.289898:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572884.289904:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.289913:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810030592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572884.289922:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572884.289927:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.289931:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810030592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572884.289955:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572884.289966:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572884.289977:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572884.289978:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572884.289980:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572884.289983:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572884.289987:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13350 00000400:00000200:0.0:1788572884.289992:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572884.289996:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572884.290001:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572884.290005:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290009:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290016:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290020:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572884.290037:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572884.290040:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572884.290044:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572884.290049:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572884.290053:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572884.290058:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572884.290064:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290076:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290083:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290088:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572884.290092:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572884.290096:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290099:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572884.290103:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572884.290111:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572884.290115:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290119:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572884.290124:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572884.290137:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290139:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572884.290144:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572884.290149:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572884.290153:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572884.290156:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290160:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572884.290163:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572884.290166:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572884.290172:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572884.290178:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.290188:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810030720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572884.290202:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572884.290205:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.290212:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810030720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572884.290228:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572884.290233:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572884.290238:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572884.290243:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572884.290247:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572884.290251:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572884.290258:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13350 00000400:00000200:0.0:1788572884.290289:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572884.290297:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572884.290300:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572884.290306:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290310:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290315:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290327:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572884.290333:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572884.290337:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572884.290339:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572884.290341:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572884.290345:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572884.290349:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572884.290356:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290361:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290367:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290371:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572884.290376:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572884.290381:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290385:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572884.290388:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572884.290393:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572884.290397:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290401:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572884.290404:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572884.290409:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290414:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572884.290421:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572884.290425:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572884.290431:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572884.290434:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290439:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572884.290442:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572884.290447:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572884.290454:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572884.290460:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.290469:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810030848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572884.290510:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572884.290514:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.290522:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810030848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572884.290539:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572884.290542:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572884.290546:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572884.290549:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572884.290551:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572884.290557:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572884.290562:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13350 00000400:00000200:0.0:1788572884.290570:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572884.290575:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572884.290577:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572884.290580:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290584:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290588:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290593:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572884.290597:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572884.290601:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572884.290603:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572884.290607:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572884.290612:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572884.290616:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572884.290622:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290628:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290632:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290663:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572884.290675:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572884.290682:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290688:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572884.290692:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572884.290697:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572884.290702:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290707:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572884.290711:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572884.290717:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290719:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572884.290722:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572884.290724:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572884.290730:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572884.290732:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290736:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572884.290739:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572884.290743:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572884.290750:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572884.290755:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.290764:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810030976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572884.290778:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572884.290782:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.290787:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810030976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572884.290815:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572884.290819:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572884.290825:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572884.290828:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572884.290831:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572884.290836:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572884.290846:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13350 00000400:00000200:0.0:1788572884.290852:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572884.290858:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572884.290863:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572884.290870:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290874:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290876:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290881:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572884.290885:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572884.290888:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572884.290892:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572884.290895:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572884.290899:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572884.290905:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572884.290912:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290917:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572884.290923:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290928:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572884.290933:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572884.290940:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572884.290946:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572884.290950:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572884.290955:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572884.290959:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290964:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572884.290968:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572884.290971:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.290973:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572884.290979:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572884.290984:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572884.290989:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572884.290998:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572884.291002:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572884.291005:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572884.291008:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572884.291015:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572884.291020:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.291042:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810031104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572884.291056:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572884.291061:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.291069:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810031104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572884.291089:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572884.291096:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13360) 00000020:00000001:0.0:1788572887.233818:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572887.233821:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572887.233824:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572888.129831:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308027102, at 4308027136 00000100:00000010:0.0:1788572888.129846:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788572888.258046:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572888.258050:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572888.258053:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572892.353956:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572892.353960:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572892.353963:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572893.249710:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308032223, at 4308032256 00000100:00000010:0.0:1788572893.249722:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788572893.377826:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572893.377829:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572893.377832:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572897.473799:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572897.473802:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572897.473805:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572898.497785:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572898.497789:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572898.497792:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572899.521779:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308038495, at 4308038528 00000100:00000010:0.0:1788572899.521810:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788572902.593906:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572902.593910:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572902.593913:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572903.617929:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572903.617937:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572903.617942:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572905.025777:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308043999, at 4308044032 00000100:00000010:0.0:1788572905.025801:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788572907.713856:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572907.713860:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572907.713862:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572908.737867:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572908.737870:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572908.737873:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788572909.377952:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788572909.377991:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572909.377999:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572909.378002:0:2390:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788572909.378005:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572909.378008:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572909.378012:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572909.378016:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788572909.378021:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572909.378025:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572909.378032:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3d48 00000400:00000010:0.0:1788572909.378035:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3d48. 00000100:00000001:0.0:1788572909.378040:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572909.378049:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68a80 x1875440810033920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378092:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572909.378097:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810033920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378104:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572909.378106:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572909.378116:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572909.378119:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788572909.378122:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572909.378126:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572909.378130:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3770 00000400:00000010:0.0:1788572909.378133:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3770. 00000100:00000001:0.0:1788572909.378136:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572909.378140:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69c00 x1875440810034048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378152:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572909.378157:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810034048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378170:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572909.378174:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572909.378186:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572909.378191:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788572909.378195:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572909.378197:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572909.378199:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3198 00000400:00000010:0.0:1788572909.378202:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3198. 00000100:00000001:0.0:1788572909.378206:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572909.378212:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68000 x1875440810034176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378221:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572909.378228:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810034176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378258:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572909.378264:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572909.378277:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572909.378281:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788572909.378284:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572909.378311:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572909.378316:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3e58 00000400:00000010:0.0:1788572909.378318:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3e58. 00000100:00000001:0.0:1788572909.378320:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572909.378326:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68e00 x1875440810034304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378341:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572909.378349:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810034304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378361:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572909.378370:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572909.378386:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572909.378391:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788572909.378397:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572909.378406:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572909.378413:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572909.378417:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572909.378422:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572909.378431:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69500 x1875440810034432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378449:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572909.378456:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810034432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572909.378472:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572909.378477:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788572910.145720:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308049118, at 4308049152 00000100:00000010:0.0:1788572910.145730:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788572912.833781:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572912.833784:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572912.833786:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572913.857801:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572913.857805:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572913.857810:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572913.921747:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572913.921757:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572913.921763:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572913.921925:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572913.921940:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572913.921945:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572913.921948:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572915.265760:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308054238, at 4308054272 00000100:00000010:0.0:1788572915.265775:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788572917.953860:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572917.953865:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572917.953869:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572918.977892:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572918.977896:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572918.977899:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572921.665768:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308060639, at 4308060672 00000100:00000010:0.0:1788572921.665787:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788572923.073859:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572923.073863:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572923.073865:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572924.097853:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572924.097856:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572924.097859:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572927.041804:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308066014, at 4308066048 00000100:00000010:0.0:1788572927.041823:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788572928.193869:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572928.193872:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572928.193875:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572929.217836:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572929.217841:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572929.217845:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572932.161773:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308071134, at 4308071168 00000100:00000010:0.0:1788572932.161788:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788572933.313853:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572933.313858:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572933.313862:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572934.337880:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572934.337887:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572934.337892:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572937.281782:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308076254, at 4308076288 00000100:00000010:0.0:1788572937.281807:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788572938.433802:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572938.433806:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572938.433809:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572938.561752:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572938.561764:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572938.561766:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572938.561770:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572938.561774:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572938.561782:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13405 00000400:00000200:0.0:1788572938.561792:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572938.561798:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572938.561801:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572938.561803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561806:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.561809:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561818:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572938.561842:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572938.561844:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572938.561846:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572938.561850:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572938.561851:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572938.561853:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572938.561855:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561857:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.561858:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561859:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572938.561862:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572938.561865:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561867:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572938.561870:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572938.561873:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572938.561875:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.561877:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572938.561879:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572938.561885:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788572938.561888:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572938.561892:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572938.561894:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.561897:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572938.561900:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572938.561905:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572938.561915:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572938.561921:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.561930:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810038528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572938.561938:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572938.561942:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.561946:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810038528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572938.561973:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572938.561976:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572938.561979:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572938.561980:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572938.561982:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572938.561983:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572938.561985:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13405 00000400:00000200:0.0:1788572938.561988:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572938.561990:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572938.561992:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572938.561994:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561996:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.561997:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.561999:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572938.562001:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572938.562002:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572938.562004:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572938.562006:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572938.562008:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572938.562009:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572938.562012:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562013:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562015:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562016:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572938.562018:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572938.562020:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562024:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572938.562031:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572938.562037:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572938.562041:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562044:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572938.562050:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572938.562056:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562060:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572938.562066:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788572938.562069:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572938.562073:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572938.562076:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562079:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572938.562082:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572938.562086:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572938.562093:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572938.562101:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.562113:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810038656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572938.562135:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572938.562138:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.562147:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810038656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572938.562158:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572938.562163:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572938.562169:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572938.562172:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572938.562174:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572938.562177:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572938.562183:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13405 00000400:00000200:0.0:1788572938.562188:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572938.562193:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572938.562195:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572938.562208:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562210:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562215:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562219:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572938.562221:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572938.562222:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572938.562223:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572938.562225:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572938.562229:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572938.562233:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572938.562236:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562239:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562245:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562247:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572938.562250:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572938.562254:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562258:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572938.562261:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572938.562265:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572938.562268:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562270:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572938.562271:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572938.562275:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562280:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572938.562285:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788572938.562290:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572938.562293:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572938.562295:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562296:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572938.562297:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572938.562301:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572938.562307:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572938.562317:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.562325:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810038784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572938.562337:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572938.562341:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.562346:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810038784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572938.562359:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572938.562363:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572938.562377:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572938.562384:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572938.562389:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572938.562396:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572938.562404:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13405 00000400:00000200:0.0:1788572938.562415:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572938.562424:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572938.562429:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572938.562435:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562438:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562443:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562447:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572938.562451:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572938.562454:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572938.562457:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572938.562459:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572938.562462:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572938.562463:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572938.562466:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562467:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562476:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562480:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572938.562484:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572938.562489:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562491:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572938.562495:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572938.562498:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572938.562500:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562502:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572938.562504:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572938.562508:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562511:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572938.562515:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788572938.562518:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572938.562523:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572938.562524:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562527:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572938.562529:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572938.562532:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572938.562537:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572938.562542:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.562549:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810038912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572938.562566:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572938.562570:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.562578:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810038912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572938.562589:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572938.562595:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572938.562600:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572938.562604:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572938.562612:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572938.562629:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572938.562632:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13405 00000400:00000200:0.0:1788572938.562666:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572938.562670:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572938.562672:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572938.562674:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562678:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562682:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562686:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572938.562691:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572938.562694:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572938.562698:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572938.562702:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572938.562709:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572938.562714:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572938.562719:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562725:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572938.562729:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562736:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572938.562742:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572938.562750:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572938.562754:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572938.562756:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572938.562760:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572938.562765:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562768:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572938.562772:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572938.562777:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562781:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572938.562792:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572938.562795:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572938.562801:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572938.562803:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.562806:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572938.562808:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572938.562811:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572938.562815:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788572938.562820:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.562827:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810039040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572938.562839:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572938.562842:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.562845:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810039040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572938.562854:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572938.562859:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13415) 00000020:00000001:0.0:1788572939.457767:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572939.457772:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572939.457776:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572939.842428:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572939.842434:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572939.842436:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572939.842438:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572939.842441:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572939.842445:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13410 00000400:00000200:0.0:1788572939.842451:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572939.842456:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572939.842459:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572939.842461:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842463:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.842465:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842475:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572939.842478:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572939.842480:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572939.842482:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572939.842485:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572939.842487:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572939.842489:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572939.842493:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842495:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.842497:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842500:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572939.842503:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572939.842507:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842510:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572939.842513:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572939.842516:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572939.842518:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.842521:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572939.842523:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572939.842528:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572939.842531:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572939.842534:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572939.842535:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.842537:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572939.842538:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572939.842540:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572939.842544:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572939.842547:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.842553:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810039296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572939.842559:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572939.842561:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.842570:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810039296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572939.842588:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572939.842590:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572939.842593:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572939.842594:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572939.842595:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572939.842597:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572939.842599:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13410 00000400:00000200:0.0:1788572939.842602:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572939.842604:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572939.842606:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572939.842609:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842611:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.842614:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842615:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572939.842617:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572939.842618:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572939.842619:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572939.842621:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572939.842622:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572939.842625:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572939.842628:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842629:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.842631:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842687:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572939.842690:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572939.842693:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842696:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572939.842708:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572939.842716:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572939.842720:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.842723:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572939.842728:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572939.842741:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.842757:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572939.842764:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572939.842771:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572939.842783:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572939.842787:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.842791:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572939.842793:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572939.842797:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572939.842804:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572939.842809:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.842817:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810039424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572939.842832:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572939.842840:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.842846:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810039424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572939.842856:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572939.842860:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572939.842863:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572939.842866:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572939.842870:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572939.842876:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572939.842882:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13410 00000400:00000200:0.0:1788572939.842895:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572939.842910:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572939.842917:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572939.842932:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842937:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.842945:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842956:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572939.842962:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572939.842964:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572939.842968:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572939.842972:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572939.842975:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572939.842978:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572939.842987:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.842999:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.843006:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843010:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572939.843020:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572939.843026:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843037:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572939.843040:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572939.843048:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572939.843055:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843063:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572939.843065:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572939.843069:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843075:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572939.843082:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572939.843087:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572939.843094:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572939.843097:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843110:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572939.843112:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572939.843118:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572939.843130:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572939.843143:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.843158:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810039552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572939.843183:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572939.843195:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.843205:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810039552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572939.843253:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572939.843257:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572939.843266:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572939.843270:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572939.843274:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572939.843286:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572939.843294:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13410 00000400:00000200:0.0:1788572939.843311:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572939.843315:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572939.843316:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572939.843320:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843325:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.843333:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843345:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572939.843357:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572939.843359:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572939.843361:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572939.843365:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572939.843367:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572939.843368:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572939.843378:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843381:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.843395:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843397:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572939.843399:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572939.843406:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843411:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572939.843423:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572939.843431:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572939.843437:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843444:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572939.843450:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572939.843462:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843472:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572939.843483:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572939.843494:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572939.843504:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572939.843509:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843521:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572939.843526:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572939.843531:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572939.843537:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572939.843548:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.843571:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810039680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572939.843624:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572939.843665:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.843674:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810039680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572939.843703:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572939.843712:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572939.843721:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572939.843723:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572939.843730:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572939.843735:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572939.843744:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13410 00000400:00000200:0.0:1788572939.843769:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572939.843777:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572939.843788:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572939.843802:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843813:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.843823:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843832:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572939.843838:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572939.843841:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572939.843845:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572939.843848:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572939.843856:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572939.843863:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572939.843878:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843886:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572939.843892:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843901:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572939.843915:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572939.843924:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572939.843927:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572939.843928:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572939.843933:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572939.843939:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843945:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572939.843950:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572939.843957:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.843961:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572939.843980:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572939.843990:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572939.844003:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572939.844005:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.844011:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572939.844014:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572939.844018:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572939.844029:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572939.844039:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.844048:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810039808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572939.844074:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572939.844082:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.844089:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810039808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572939.844140:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572939.844153:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13416) 00000020:00000001:0.0:1788572943.553793:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572943.553797:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572943.553799:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572943.681744:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308082655, at 4308082688 00000100:00000010:0.0:1788572943.681758:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788572944.577862:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572944.577866:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572944.577869:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572948.673854:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572948.673860:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572948.673864:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572949.057710:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308088030, at 4308088064 00000100:00000010:0.0:1788572949.057729:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788572949.697867:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572949.697873:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572949.697877:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572951.809719:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572951.809727:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572951.809731:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572951.809883:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572951.809896:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572951.809902:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572951.809905:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572953.793865:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572953.793869:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572953.793872:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572954.177712:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308093150, at 4308093184 00000100:00000010:0.0:1788572954.177720:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788572954.817837:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572954.817841:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572954.817844:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572958.913832:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572958.913837:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572958.913840:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572959.297747:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308098271, at 4308098304 00000100:00000010:0.0:1788572959.297762:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788572959.937808:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572959.937812:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572959.937815:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572960.577791:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572960.577803:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572960.577806:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572960.577810:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572960.577816:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572960.577824:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13427 00000400:00000200:0.0:1788572960.577834:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572960.577841:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572960.577844:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572960.577848:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.577871:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.577874:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.577876:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572960.577881:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572960.577883:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572960.577886:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572960.577900:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572960.577903:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572960.577906:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572960.577910:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.577912:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.577914:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.577916:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572960.577919:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572960.577923:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572960.577926:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572960.577929:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572960.577933:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572960.577937:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.577939:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572960.577942:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572960.577949:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572960.577953:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572960.577958:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572960.577960:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.577963:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572960.577966:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572960.577971:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572960.577981:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788572960.577989:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.577999:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810041856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572960.578023:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572960.578028:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.578035:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810041856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572960.578061:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572960.578065:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572960.578070:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572960.578071:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572960.578081:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572960.578085:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572960.578089:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13427 00000400:00000200:0.0:1788572960.578097:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572960.578105:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572960.578110:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572960.578118:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578128:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.578136:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578145:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572960.578153:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572960.578158:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572960.578162:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572960.578167:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572960.578173:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572960.578177:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572960.578184:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578192:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.578200:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578206:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572960.578222:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572960.578231:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578240:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572960.578245:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572960.578256:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572960.578263:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578270:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572960.578278:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572960.578287:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578293:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572960.578301:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572960.578309:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572960.578319:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572960.578324:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578330:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572960.578336:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572960.578342:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572960.578352:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788572960.578362:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.578372:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810041984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572960.578386:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572960.578392:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.578404:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810041984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572960.578424:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572960.578431:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572960.578437:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572960.578441:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572960.578445:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572960.578452:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572960.578471:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13427 00000400:00000200:0.0:1788572960.578482:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572960.578491:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572960.578496:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572960.578508:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578513:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.578517:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578523:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572960.578528:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572960.578533:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572960.578535:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572960.578538:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572960.578544:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572960.578547:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572960.578553:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578561:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.578571:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578591:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572960.578599:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572960.578611:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578619:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572960.578626:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572960.578685:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572960.578692:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578700:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572960.578707:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572960.578714:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578718:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572960.578723:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572960.578731:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572960.578739:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572960.578753:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.578760:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572960.578764:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572960.578770:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572960.578781:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788572960.578789:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.578803:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810042112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572960.578823:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572960.578830:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.578844:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810042112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572960.578865:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572960.578874:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572960.578885:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572960.578892:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572960.578897:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572960.578904:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572960.578908:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13427 00000400:00000200:0.0:1788572960.578921:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572960.578932:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572960.578939:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572960.578949:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578956:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.578963:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.578971:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572960.578977:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572960.578983:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572960.578989:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572960.578995:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572960.579004:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572960.579019:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572960.579030:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579037:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.579046:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579055:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572960.579064:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572960.579071:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579076:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572960.579081:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572960.579088:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572960.579094:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579099:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572960.579104:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572960.579111:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579116:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572960.579123:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572960.579128:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572960.579133:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572960.579137:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579143:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572960.579149:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572960.579155:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572960.579166:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788572960.579174:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.579189:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810042240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572960.579208:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572960.579213:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.579226:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810042240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572960.579262:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572960.579278:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572960.579284:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572960.579291:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572960.579296:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572960.579307:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572960.579318:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13427 00000400:00000200:0.0:1788572960.579333:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572960.579345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572960.579353:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572960.579364:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579373:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.579381:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579390:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572960.579398:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572960.579404:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572960.579411:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572960.579416:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572960.579422:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572960.579430:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572960.579442:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579451:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572960.579460:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579467:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572960.579475:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572960.579487:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572960.579496:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572960.579503:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572960.579510:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572960.579518:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579524:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572960.579536:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572960.579541:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579546:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572960.579555:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572960.579563:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572960.579571:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572960.579592:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.579599:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572960.579603:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572960.579610:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572960.579621:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788572960.579631:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.579673:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5c00 x1875440810042368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572960.579700:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572960.579707:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.579723:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5c00 x1875440810042368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572960.579749:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572960.579759:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13437) 00000020:00000001:0.0:1788572964.033792:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572964.033795:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572964.033798:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572965.057826:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572965.057832:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572965.057836:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572965.377782:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308104351, at 4308104384 00000100:00000010:0.0:1788572965.377802:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788572969.153812:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572969.153815:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572969.153818:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572970.177864:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572970.177867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572970.177886:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572970.561746:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308109534, at 4308109568 00000100:00000010:0.0:1788572970.561762:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788572974.273931:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572974.273936:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572974.273939:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788572974.913865:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788572974.913905:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572974.913914:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572974.913917:0:2391:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788572974.913921:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572974.913924:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572974.913933:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572974.913937:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788572974.913943:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572974.913948:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572974.913955:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788572974.913958:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788572974.913964:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572974.913973:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6b800 x1875440810043904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.913996:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572974.914005:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440810043904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914013:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.914015:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572974.914025:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572974.914027:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788572974.914031:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572974.914034:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572974.914037:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788572974.914051:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788572974.914055:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572974.914059:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68380 x1875440810044032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914068:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572974.914071:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810044032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914079:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.914081:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572974.914087:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572974.914089:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788572974.914092:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572974.914096:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572974.914099:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3440 00000400:00000010:0.0:1788572974.914101:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3440. 00000100:00000001:0.0:1788572974.914103:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572974.914107:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6bb80 x1875440810044160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914115:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572974.914125:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810044160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914133:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.914134:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572974.914140:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572974.914147:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9e00. 00000400:00000200:0.0:1788572974.914150:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572974.914154:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572974.914157:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788572974.914158:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788572974.914160:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572974.914166:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6b480 x1875440810044288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914187:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572974.914190:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810044288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914197:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.914203:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572974.914209:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788572974.914212:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788572974.914215:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572974.914219:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572974.914222:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788572974.914224:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788572974.914227:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572974.914231:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6aa00 x1875440810044416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914240:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572974.914244:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810044416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572974.914251:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.914252:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788572975.297886:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572975.297890:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572975.297892:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572975.681740:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308114654, at 4308114688 00000100:00000010:0.0:1788572975.681754:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788572979.393769:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572979.393773:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572979.393776:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572980.417855:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572980.417860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572980.417865:0:243062:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788572980.417871:0:243062:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788572980.417880:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.417884:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000545753e5/0x1441586093168476 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5269:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a2244 expref: -99 pid: 261328 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.417901:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.417903:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.417907:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.417909:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.417912:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.417915:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.417915:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.417917:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.417918:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.417920:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.417922:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.417924:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.417925:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.417928:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.417931:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac380 -> ffff9c371ee2edc8 00000020:00000001:0.0:1788572980.417933:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2edc8 -> ffff9c371ee2edc8 00000020:00000001:0.0:1788572980.417934:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.417936:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.417938:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdb0:0x0] ignore/verify layout 1/0, layout version 3 restore needed 0 00000020:00000001:0.0:1788572980.417941:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.417942:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.417943:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.417947:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdb0:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.417950:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.417952:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.417953:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.417955:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.417956:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.417957:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.417957:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.417958:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.417959:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.417961:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.417966:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.417968:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.417970:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.417971:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.417973:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.417974:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdb0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.417976:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.417977:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.417977:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.417979:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.417981:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.417982:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac380 -> ffff9c371ee2edc8 00010000:00000001:0.0:1788572980.417984:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.417986:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.417988:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac380 -> ffff9c371ee2edc8 00000020:00000001:0.0:1788572980.417989:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.417991:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.417994:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418002:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418004:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418005:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac380 -> ffff9c371ee2edc8 00000020:00000001:0.0:1788572980.418006:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2edc8 -> ffff9c371ee2edc8 00000020:00000001:0.0:1788572980.418007:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418009:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418011:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d7ac380 -> ffff9c371ee2edc8 00000008:00000001:0.0:1788572980.418013:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418015:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418016:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418018:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418021:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418022:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418023:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418025:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418027:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204a600 with handle 0x1441586093168476 from hash 00010000:00000001:0.0:1788572980.418029:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418030:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418032:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418032:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418038:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418040:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418041:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418044:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001ab58db4/0x144158609316845a lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5268:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a21fe expref: -99 pid: 261324 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418049:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418050:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418052:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418053:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418054:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418055:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418056:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418057:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418058:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418060:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418061:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418062:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418064:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418065:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418066:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8ba40 -> ffff9c37184593a0 00000020:00000001:0.0:1788572980.418067:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37184593a0 -> ffff9c37184593a0 00000020:00000001:0.0:1788572980.418068:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418069:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418070:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdae:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418071:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418072:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418073:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418075:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdae:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418077:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418077:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418079:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418080:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418081:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418082:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418082:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418087:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418088:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418090:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418092:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418093:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418095:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418096:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418097:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418099:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdae:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418102:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418103:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418104:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418106:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418108:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418110:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8ba40 -> ffff9c37184593a0 00010000:00000001:0.0:1788572980.418112:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418114:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418116:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8ba40 -> ffff9c37184593a0 00000020:00000001:0.0:1788572980.418118:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418119:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418121:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418123:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418125:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418127:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8ba40 -> ffff9c37184593a0 00000020:00000001:0.0:1788572980.418129:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37184593a0 -> ffff9c37184593a0 00000020:00000001:0.0:1788572980.418130:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418132:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418134:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8ba40 -> ffff9c37184593a0 00000008:00000001:0.0:1788572980.418137:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418139:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418141:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418143:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418145:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418146:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418147:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418149:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418150:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372ee41000 with handle 0x144158609316845a from hash 00010000:00000001:0.0:1788572980.418152:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418157:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418158:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418159:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418160:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418161:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418163:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418165:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000f9053d9f/0x144158609316843e lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5267:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a21b8 expref: -99 pid: 261320 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418179:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418183:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418184:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418185:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418187:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418188:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418189:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418190:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418191:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418192:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418193:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418195:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418196:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418198:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418199:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8380 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572980.418200:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e210 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572980.418201:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418202:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418203:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdac:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418204:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418205:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418206:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418208:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdac:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418209:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418210:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418212:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418213:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418217:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418218:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418219:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418220:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418221:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418222:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418223:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418224:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418225:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418226:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418227:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418228:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdac:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418229:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418230:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418231:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418232:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418233:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418234:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8380 -> ffff9c371ee2e210 00010000:00000001:0.0:1788572980.418236:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418237:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418238:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8380 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572980.418239:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418240:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418241:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418243:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418244:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418245:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8380 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572980.418246:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e210 -> ffff9c371ee2e210 00000020:00000001:0.0:1788572980.418247:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418248:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418249:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8380 -> ffff9c371ee2e210 00000008:00000001:0.0:1788572980.418251:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418252:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418253:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418254:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418256:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418257:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418258:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418260:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418265:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372eb6a600 with handle 0x144158609316843e from hash 00010000:00000001:0.0:1788572980.418267:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418268:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418269:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418269:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418271:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418272:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418274:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418276:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000aa77ba1e/0x1441586093168422 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5266:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a2172 expref: -99 pid: 261316 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418282:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418283:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418284:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418285:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418286:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418288:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418289:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418289:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418291:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418292:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418293:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418294:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418295:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418296:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418297:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09ba40 -> ffff9c3732162148 00000020:00000001:0.0:1788572980.418298:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3732162148 -> ffff9c3732162148 00000020:00000001:0.0:1788572980.418299:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418300:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418301:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xdaa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418303:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418304:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418304:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418322:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xdaa:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418325:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418331:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418333:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418335:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418336:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418337:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418338:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418339:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418339:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418341:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418342:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418343:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418344:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418344:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418345:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418346:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xdaa:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418348:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418349:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418350:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418351:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418352:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418354:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09ba40 -> ffff9c3732162148 00010000:00000001:0.0:1788572980.418355:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418356:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418358:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09ba40 -> ffff9c3732162148 00000020:00000001:0.0:1788572980.418359:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418360:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418361:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418362:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418363:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418364:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09ba40 -> ffff9c3732162148 00000020:00000001:0.0:1788572980.418365:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3732162148 -> ffff9c3732162148 00000020:00000001:0.0:1788572980.418366:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418367:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418368:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09ba40 -> ffff9c3732162148 00000008:00000001:0.0:1788572980.418370:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418371:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418372:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418373:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418374:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418378:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418379:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418380:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418382:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f011400 with handle 0x1441586093168422 from hash 00010000:00000001:0.0:1788572980.418383:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418384:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418385:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418386:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418387:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418388:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418389:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418391:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000004a018978/0x1441586093168406 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5265:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a212c expref: -99 pid: 261312 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418396:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418397:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418398:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418400:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418401:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418402:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418403:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418403:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418404:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418406:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418406:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418408:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418409:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418410:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418411:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c96c0 -> ffff9c371ee2ef58 00000020:00000001:0.0:1788572980.418412:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ef58 -> ffff9c371ee2ef58 00000020:00000001:0.0:1788572980.418413:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418414:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418415:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xda8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418417:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418417:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418421:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418423:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xda8:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418425:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418426:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418427:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418428:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418429:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418430:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418430:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418431:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418432:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418433:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418434:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418435:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418435:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418436:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418437:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418438:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418439:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418440:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418440:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418441:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418442:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418444:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c96c0 -> ffff9c371ee2ef58 00010000:00000001:0.0:1788572980.418445:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418446:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418447:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c96c0 -> ffff9c371ee2ef58 00000020:00000001:0.0:1788572980.418448:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418449:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418450:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418451:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418452:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418453:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c96c0 -> ffff9c371ee2ef58 00000020:00000001:0.0:1788572980.418454:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ef58 -> ffff9c371ee2ef58 00000020:00000001:0.0:1788572980.418454:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418456:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418457:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c96c0 -> ffff9c371ee2ef58 00000008:00000001:0.0:1788572980.418458:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418459:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418463:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418465:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418466:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418467:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418468:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418469:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418470:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f010a00 with handle 0x1441586093168406 from hash 00010000:00000001:0.0:1788572980.418472:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418472:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418473:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418474:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418475:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418476:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418477:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418479:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001b7036a2/0x14415860931683ea lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5264:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a20e6 expref: -99 pid: 261308 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418483:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418483:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418484:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418485:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418486:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418487:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418488:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418489:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418490:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418491:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418492:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418493:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418494:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418495:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418496:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9500 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572980.418498:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ec38 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572980.418499:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418500:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418501:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xda6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418528:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418529:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418530:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418532:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xda6:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418534:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418535:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418537:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418539:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418540:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418542:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418543:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418544:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418545:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418547:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418549:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418551:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418552:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418553:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418555:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418556:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418559:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418560:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418561:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418563:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418566:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418568:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9500 -> ffff9c371ee2ec38 00010000:00000001:0.0:1788572980.418570:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418572:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418573:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9500 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572980.418575:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418577:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418578:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418580:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418582:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418584:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9500 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572980.418586:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2ec38 -> ffff9c371ee2ec38 00000020:00000001:0.0:1788572980.418587:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418589:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418596:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9500 -> ffff9c371ee2ec38 00000008:00000001:0.0:1788572980.418598:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418599:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418601:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418602:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418604:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418605:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418606:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418607:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418609:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f010c00 with handle 0x14415860931683ea from hash 00010000:00000001:0.0:1788572980.418611:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418612:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418612:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418613:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418614:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418615:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418617:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418618:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000043229ee0/0x14415860931683ce lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5263:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a20a0 expref: -99 pid: 261304 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418623:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418624:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418626:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418627:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418628:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418629:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418630:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418631:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418632:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418633:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418659:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418661:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418663:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418664:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418665:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9180 -> ffff9c371ee2e2d8 00000020:00000001:0.0:1788572980.418671:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e2d8 -> ffff9c371ee2e2d8 00000020:00000001:0.0:1788572980.418673:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418674:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418675:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xda4:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418677:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418677:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418678:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418680:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xda4:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418682:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418683:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418684:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418685:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418685:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418686:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418687:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418688:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418688:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418689:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418691:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418692:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418692:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418693:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418694:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418695:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418696:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418697:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418698:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418699:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418700:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418701:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9180 -> ffff9c371ee2e2d8 00010000:00000001:0.0:1788572980.418703:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418704:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418705:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9180 -> ffff9c371ee2e2d8 00000020:00000001:0.0:1788572980.418706:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418707:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418708:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418710:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418711:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418712:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9180 -> ffff9c371ee2e2d8 00000020:00000001:0.0:1788572980.418717:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371ee2e2d8 -> ffff9c371ee2e2d8 00000020:00000001:0.0:1788572980.418718:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418719:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418721:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9180 -> ffff9c371ee2e2d8 00000008:00000001:0.0:1788572980.418723:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418724:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418725:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418726:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418728:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418729:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418730:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418731:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418733:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f011200 with handle 0x14415860931683ce from hash 00010000:00000001:0.0:1788572980.418734:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418735:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418736:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418736:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418737:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418738:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418740:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418741:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000012528090/0x14415860931683b2 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5262:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a205a expref: -99 pid: 261300 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418746:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418747:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418748:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418749:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418750:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418752:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418752:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418753:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418754:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418755:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418756:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418758:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418762:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418763:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418765:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37053301c0 -> ffff9c37031ff5f8 00000020:00000001:0.0:1788572980.418766:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff5f8 -> ffff9c37031ff5f8 00000020:00000001:0.0:1788572980.418767:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418768:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418769:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xda2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418770:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418771:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418771:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418773:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xda2:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418775:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418775:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418777:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418778:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418778:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418779:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418780:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418780:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418781:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418782:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418783:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418784:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418785:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418785:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418786:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418787:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418789:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418790:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418791:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418792:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418793:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418794:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37053301c0 -> ffff9c37031ff5f8 00010000:00000001:0.0:1788572980.418795:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418796:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418797:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37053301c0 -> ffff9c37031ff5f8 00000020:00000001:0.0:1788572980.418798:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418799:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418800:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418804:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418805:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418806:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37053301c0 -> ffff9c37031ff5f8 00000020:00000001:0.0:1788572980.418807:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37031ff5f8 -> ffff9c37031ff5f8 00000020:00000001:0.0:1788572980.418809:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418810:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418811:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37053301c0 -> ffff9c37031ff5f8 00000008:00000001:0.0:1788572980.418812:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418813:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418814:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418816:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418817:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418817:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418818:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418819:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418821:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c371f011800 with handle 0x14415860931683b2 from hash 00010000:00000001:0.0:1788572980.418822:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418823:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418823:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418824:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418825:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418826:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418827:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418829:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000006555ea31/0x1441586093168396 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5261:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a2014 expref: -99 pid: 261296 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418833:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418834:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418834:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418835:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418836:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418838:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418838:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418839:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418840:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418841:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418845:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418847:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418848:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418850:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418851:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9a40 -> ffff9c37313e1148 00000020:00000001:0.0:1788572980.418852:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1148 -> ffff9c37313e1148 00000020:00000001:0.0:1788572980.418853:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418854:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418855:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xda0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418856:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418856:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418857:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418859:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xda0:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418860:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418861:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418862:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418863:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418864:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418864:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418865:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418866:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418866:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418867:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418868:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418869:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418870:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418871:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418871:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418872:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xda0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418874:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418874:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418875:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418876:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418877:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418878:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9a40 -> ffff9c37313e1148 00010000:00000001:0.0:1788572980.418879:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.418880:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418881:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9a40 -> ffff9c37313e1148 00000020:00000001:0.0:1788572980.418885:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.418886:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.418887:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.418888:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418890:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418891:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9a40 -> ffff9c37313e1148 00000020:00000001:0.0:1788572980.418892:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1148 -> ffff9c37313e1148 00000020:00000001:0.0:1788572980.418893:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.418894:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418895:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c9a40 -> ffff9c37313e1148 00000008:00000001:0.0:1788572980.418896:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418897:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418898:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.418900:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.418901:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.418901:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.418902:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.418903:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.418905:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c37320efa00 with handle 0x1441586093168396 from hash 00010000:00000001:0.0:1788572980.418906:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.418907:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.418907:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.418908:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.418909:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.418910:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.418911:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.418913:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.418917:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.418917:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418918:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.418920:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.418921:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.418923:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.418924:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.418929:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.418932:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418934:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.418936:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.418938:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.418940:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418942:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.418944:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572980.418946:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37184596c0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572980.418948:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.418949:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.418950:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd9e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.418953:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418954:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.418955:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.418958:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd9e:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.418961:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.418962:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.418964:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.418965:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.418967:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418968:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.418969:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.418970:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.418972:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.418974:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.418976:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.418977:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418979:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.418980:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.418981:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.418983:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd9e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.418985:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.418986:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.418988:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.418989:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.418992:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.418994:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00010000:00000001:0.0:1788572980.419008:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.419010:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419012:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572980.419013:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.419014:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.419016:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.419017:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419018:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419019:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572980.419020:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37184596c0 -> ffff9c37184596c0 00000020:00000001:0.0:1788572980.419021:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.419022:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419023:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3713d8bdc0 -> ffff9c37184596c0 00000008:00000001:0.0:1788572980.419025:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419026:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419027:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.419029:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.419030:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.419032:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.419033:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.419034:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.419036:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204bc00 with handle 0x144158609316830a from hash 00010000:00000001:0.0:1788572980.419037:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.419038:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.419039:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.419040:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.419041:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.419042:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.419044:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.419046:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000573e3650/0x14415860931682ee lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a1dc8 expref: -99 pid: 261286 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.419051:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.419052:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419053:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.419054:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419060:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.419061:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.419062:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419063:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.419065:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419066:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.419067:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.419069:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419070:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419071:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.419072:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8000 -> ffff9c37313e1530 00000020:00000001:0.0:1788572980.419073:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1530 -> ffff9c37313e1530 00000020:00000001:0.0:1788572980.419074:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.419075:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.419076:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd9c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.419078:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419078:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.419079:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.419081:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd9c:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.419083:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.419083:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.419085:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.419086:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.419087:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419089:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419090:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.419091:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.419093:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419094:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.419096:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.419098:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419099:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419100:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.419102:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.419103:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd9c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.419106:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.419107:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.419108:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419114:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419117:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419119:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8000 -> ffff9c37313e1530 00010000:00000001:0.0:1788572980.419122:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.419123:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419125:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8000 -> ffff9c37313e1530 00000020:00000001:0.0:1788572980.419127:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.419128:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.419130:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.419132:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419134:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419136:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8000 -> ffff9c37313e1530 00000020:00000001:0.0:1788572980.419138:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1530 -> ffff9c37313e1530 00000020:00000001:0.0:1788572980.419139:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.419141:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419143:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8000 -> ffff9c37313e1530 00000008:00000001:0.0:1788572980.419145:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419147:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419149:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.419151:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.419153:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.419154:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.419156:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.419158:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.419160:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c37320ef400 with handle 0x14415860931682ee from hash 00010000:00000001:0.0:1788572980.419162:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.419164:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.419165:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.419166:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.419168:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.419170:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.419172:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.419175:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000fed4281f/0x14415860931682d2 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a1d82 expref: -99 pid: 261282 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.419184:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.419191:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419192:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.419195:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419197:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.419199:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.419200:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419201:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.419203:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419205:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.419207:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.419209:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419210:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419212:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.419213:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8a80 -> ffff9c37313e1c38 00000020:00000001:0.0:1788572980.419214:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1c38 -> ffff9c37313e1c38 00000020:00000001:0.0:1788572980.419216:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.419217:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.419218:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd9a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.419219:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419220:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.419221:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.419222:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd9a:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.419224:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.419225:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.419226:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.419227:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.419228:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419229:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419230:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.419230:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.419231:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419232:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.419234:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.419235:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419235:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419236:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.419237:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.419238:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd9a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.419245:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.419245:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.419246:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419248:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419249:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419250:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8a80 -> ffff9c37313e1c38 00010000:00000001:0.0:1788572980.419252:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.419253:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419254:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8a80 -> ffff9c37313e1c38 00000020:00000001:0.0:1788572980.419255:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.419256:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.419257:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.419258:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419260:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419261:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8a80 -> ffff9c37313e1c38 00000020:00000001:0.0:1788572980.419262:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e1c38 -> ffff9c37313e1c38 00000020:00000001:0.0:1788572980.419263:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.419264:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419265:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8a80 -> ffff9c37313e1c38 00000008:00000001:0.0:1788572980.419267:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419268:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419269:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.419270:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.419272:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.419272:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.419273:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.419275:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.419277:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204b000 with handle 0x14415860931682d2 from hash 00010000:00000001:0.0:1788572980.419278:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.419279:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.419280:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.419281:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.419282:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.419283:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.419284:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.419286:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000002e1c4230/0x14415860931682af lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a1c40 expref: -99 pid: 261276 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.419298:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.419299:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419300:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.419301:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419302:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.419303:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.419304:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419305:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.419306:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419307:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.419308:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.419310:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419311:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419312:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.419313:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8c40 -> ffff9c37313e13a0 00000020:00000001:0.0:1788572980.419314:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e13a0 -> ffff9c37313e13a0 00000020:00000001:0.0:1788572980.419315:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.419316:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.419317:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd98:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.419318:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419319:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.419319:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.419321:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd98:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.419322:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.419323:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.419324:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.419325:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.419326:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419326:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419327:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.419328:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.419328:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419329:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.419330:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.419331:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419335:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419335:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.419336:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.419338:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd98:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.419339:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.419340:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.419341:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419342:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419343:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419344:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8c40 -> ffff9c37313e13a0 00010000:00000001:0.0:1788572980.419345:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.419346:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419347:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8c40 -> ffff9c37313e13a0 00000020:00000001:0.0:1788572980.419348:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.419349:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.419350:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.419351:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419352:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419353:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8c40 -> ffff9c37313e13a0 00000020:00000001:0.0:1788572980.419354:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c37313e13a0 -> ffff9c37313e13a0 00000020:00000001:0.0:1788572980.419355:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.419356:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419357:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8c40 -> ffff9c37313e13a0 00000008:00000001:0.0:1788572980.419358:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419359:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419360:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.419361:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.419362:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.419363:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.419364:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.419365:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.419366:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c373204ac00 with handle 0x14415860931682af from hash 00010000:00000001:0.0:1788572980.419368:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.419368:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.419369:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.419370:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.419371:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.419372:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.419376:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.419378:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788572980.419382:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.419383:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419384:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.419385:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419386:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.419387:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.419388:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419389:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.419390:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419391:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.419392:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.419393:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419394:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419395:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.419396:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000020:00000001:0.0:1788572980.419397:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3732162c38 -> ffff9c3732162c38 00000020:00000001:0.0:1788572980.419398:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.419399:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.419400:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xd96:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.419401:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419402:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.419402:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.419404:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xd96:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.419406:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.419406:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.419408:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.419409:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.419410:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419411:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419411:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.419412:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.419413:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419417:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788572980.419419:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.419420:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419421:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.419421:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.419422:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.419423:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xd96:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.419425:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.419426:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.419426:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419427:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419428:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419430:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00010000:00000001:0.0:1788572980.419431:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.419432:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419433:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000020:00000001:0.0:1788572980.419434:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.419434:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788572980.419435:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.419436:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419438:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419438:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000020:00000001:0.0:1788572980.419439:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3732162c38 -> ffff9c3732162c38 00000020:00000001:0.0:1788572980.419440:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.419441:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419442:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c372d09b340 -> ffff9c3732162c38 00000008:00000001:0.0:1788572980.419444:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.419445:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419446:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.419447:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.419448:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.419449:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.419449:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.419451:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.419452:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c37421ca000 with handle 0x144158609316825b from hash 00010000:00000001:0.0:1788572980.419453:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.419454:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.419455:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.419459:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.419460:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.419461:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.419462:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.419464:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000e69e32cb/0x1441586093165185 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d9992484 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00000001:0.0:1788572980.419468:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.419469:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419470:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.419471:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.419472:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.419473:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.419474:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.419475:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.419476:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.419477:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.419478:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.419479:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419480:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419481:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.419482:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8700 -> ffff9c3720b2e148 00000020:00000001:0.0:1788572980.419483:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e148 -> ffff9c3720b2e148 00000020:00000001:0.0:1788572980.419484:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.419485:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.419486:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xc88:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.419487:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419488:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.419488:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.419490:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xc88:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.419491:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.419492:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.419493:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.419494:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.419495:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419496:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.419499:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.419500:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.419501:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419502:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000020:00000001:0.0:1788572980.419517:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419519:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419520:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419521:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419522:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419522:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419523:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419524:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419525:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419526:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419527:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419528:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419528:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419529:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419530:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419530:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419531:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419532:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419533:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419533:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419534:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419535:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419536:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419537:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419538:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419539:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.419540:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.419541:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000008:00000001:0.0:1788572980.419544:0:243062:0:(osc_lock.c:1343:osc_obj_dlmlock_at_pgoff()) Process entered 00000008:00000001:0.0:1788572980.419547:0:243062:0:(osc_request.c:3331:osc_match_base()) Process entered 00010000:00000001:0.0:1788572980.419549:0:243062:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572980.419555:0:243062:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37421e3600 count: 2 00010000:00000001:0.0:1788572980.419559:0:243062:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.419565:0:243062:0:(osc_request.c:3346:osc_match_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419567:0:243062:0:(osc_lock.c:1377:osc_obj_dlmlock_at_pgoff()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419570:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419573:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419574:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419577:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419583:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419585:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419589:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000c87c6eca[2 000000006e2ad75d 1 1 000000003e6fbc71 0 0 0] 00000020:00000001:0.0:1788572980.419590:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419592:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419593:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419594:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419596:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419597:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419598:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419599:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419600:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419602:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000c87c6eca[2 000000006e2ad75d 2 1 0000000000000000 0 0 0] 00000020:00000001:0.0:1788572980.419603:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419603:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419605:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419606:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace680 00000008:00000001:0.0:1788572980.419607:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419608:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace690 page ffff9c371eace680 at index 0. 00000008:00000001:0.0:1788572980.419610:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419614:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419616:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84921900 index 0 00000020:00000001:0.0:1788572980.419619:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419620:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419630:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419631:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419631:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419632:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419633:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419649:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419650:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419652:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419653:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419654:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419656:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000d94249a2[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 1] 00000020:00000001:0.0:1788572980.419657:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419658:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419658:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419659:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419660:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419664:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419665:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419666:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419667:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419669:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000d94249a2[2 000000006e2ad75d 2 1 0000000000000000 0 1 1] 00000020:00000001:0.0:1788572980.419670:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419671:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419672:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419672:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eacebc0 00000008:00000001:0.0:1788572980.419673:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419674:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eacebd0 page ffff9c371eacebc0 at index 1. 00000008:00000001:0.0:1788572980.419675:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419677:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419678:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84574280 index 1 00000020:00000001:0.0:1788572980.419679:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419680:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419682:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419683:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419684:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419685:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419686:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419687:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419688:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419689:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419689:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419690:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419692:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000dc93ebd[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 2] 00000020:00000001:0.0:1788572980.419693:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419693:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419694:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419695:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419696:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419696:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419697:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419698:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419698:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419700:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000dc93ebd[2 000000006e2ad75d 2 1 0000000000000000 0 1 2] 00000020:00000001:0.0:1788572980.419701:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419701:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419702:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419705:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace760 00000008:00000001:0.0:1788572980.419706:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419707:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace770 page ffff9c371eace760 at index 2. 00000008:00000001:0.0:1788572980.419709:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419711:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419712:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8437bac0 index 2 00000020:00000001:0.0:1788572980.419713:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419713:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419715:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419716:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419716:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419717:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419718:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419719:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419719:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419720:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419721:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419722:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419723:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000eaa488d2[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 3] 00000020:00000001:0.0:1788572980.419724:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419725:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419726:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419726:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419727:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419728:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419729:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419729:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419730:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419731:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000eaa488d2[2 000000006e2ad75d 2 1 0000000000000000 0 1 3] 00000020:00000001:0.0:1788572980.419732:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419733:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419734:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419734:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eacea00 00000008:00000001:0.0:1788572980.419735:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419736:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eacea10 page ffff9c371eacea00 at index 3. 00000008:00000001:0.0:1788572980.419737:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419738:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419739:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84a15e00 index 3 00000020:00000001:0.0:1788572980.419740:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419743:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419745:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419745:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419746:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419747:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419747:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419748:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419749:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419750:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419751:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419751:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419753:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000af683df1[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 4] 00000020:00000001:0.0:1788572980.419754:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419755:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419755:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419756:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419757:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419758:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419758:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419759:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419760:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419761:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000af683df1[2 000000006e2ad75d 2 1 0000000000000000 0 1 4] 00000020:00000001:0.0:1788572980.419762:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419763:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419763:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419764:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132c680 00000008:00000001:0.0:1788572980.419765:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419765:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132c690 page ffff9c373132c680 at index 4. 00000008:00000001:0.0:1788572980.419767:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419768:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419769:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec845c4e40 index 4 00000020:00000001:0.0:1788572980.419770:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419770:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419771:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419772:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419773:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419773:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419774:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419775:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419776:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419777:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419781:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419782:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419784:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000005b41729e[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 5] 00000020:00000001:0.0:1788572980.419784:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419785:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419786:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419786:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419787:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419788:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419789:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419789:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419790:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419792:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000005b41729e[2 000000006e2ad75d 2 1 0000000000000000 0 1 5] 00000020:00000001:0.0:1788572980.419792:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419793:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419794:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419794:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132cbc0 00000008:00000001:0.0:1788572980.419795:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419796:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132cbd0 page ffff9c373132cbc0 at index 5. 00000008:00000001:0.0:1788572980.419797:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419799:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419799:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec843a0140 index 5 00000020:00000001:0.0:1788572980.419800:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419801:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419802:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419803:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419804:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419804:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419805:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419806:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419807:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419808:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419808:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419809:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419811:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000001b093ab6[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 6] 00000020:00000001:0.0:1788572980.419811:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419812:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419813:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419813:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419814:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419818:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419819:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419819:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419820:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419822:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000001b093ab6[2 000000006e2ad75d 2 1 0000000000000000 0 1 6] 00000020:00000001:0.0:1788572980.419822:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419823:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419824:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419824:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132ca00 00000008:00000001:0.0:1788572980.419825:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419826:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132ca10 page ffff9c373132ca00 at index 6. 00000008:00000001:0.0:1788572980.419827:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419829:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419829:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8475c280 index 6 00000020:00000001:0.0:1788572980.419831:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419831:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419833:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419833:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419834:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419835:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419835:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419836:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419837:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419838:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419839:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419839:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419841:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000534527b3[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 7] 00000020:00000001:0.0:1788572980.419842:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419842:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419843:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419844:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419845:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419845:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419846:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419847:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419847:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419849:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000534527b3[2 000000006e2ad75d 2 1 0000000000000000 0 1 7] 00000020:00000001:0.0:1788572980.419850:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419850:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419851:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419854:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132c760 00000008:00000001:0.0:1788572980.419855:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419856:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132c770 page ffff9c373132c760 at index 7. 00000008:00000001:0.0:1788572980.419857:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419858:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419859:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec844d9a00 index 7 00000020:00000001:0.0:1788572980.419860:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419861:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419863:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419863:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419864:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419864:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419865:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419866:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419867:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419868:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419868:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419869:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419871:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b4e56d35[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 8] 00000020:00000001:0.0:1788572980.419872:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419872:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419873:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419874:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419875:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419875:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419876:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419877:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419877:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419879:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b4e56d35[2 000000006e2ad75d 2 1 0000000000000000 0 1 8] 00000020:00000001:0.0:1788572980.419880:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419880:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419881:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419881:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132c220 00000008:00000001:0.0:1788572980.419882:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419883:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132c230 page ffff9c373132c220 at index 8. 00000008:00000001:0.0:1788572980.419884:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419885:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419886:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84911140 index 8 00000020:00000001:0.0:1788572980.419887:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419890:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419892:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419893:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419893:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419894:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419894:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419895:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419896:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419897:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419898:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419898:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419900:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000d6fcde9e[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 9] 00000020:00000001:0.0:1788572980.419901:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419902:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419902:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419903:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419904:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419905:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419905:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419906:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419907:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419908:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000d6fcde9e[2 000000006e2ad75d 2 1 0000000000000000 0 1 9] 00000020:00000001:0.0:1788572980.419909:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419910:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419910:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419911:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132c300 00000008:00000001:0.0:1788572980.419912:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419913:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132c310 page ffff9c373132c300 at index 9. 00000008:00000001:0.0:1788572980.419914:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419915:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419916:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8439f240 index 9 00000020:00000001:0.0:1788572980.419917:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419918:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419919:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419919:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419920:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419921:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419921:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419922:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419923:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419924:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419927:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419928:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419930:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b05f2f1f[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 10] 00000020:00000001:0.0:1788572980.419930:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419931:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419932:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419933:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419934:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419934:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419935:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419935:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419936:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419938:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b05f2f1f[2 000000006e2ad75d 2 1 0000000000000000 0 1 10] 00000020:00000001:0.0:1788572980.419938:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419939:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419940:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419941:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132cd80 00000008:00000001:0.0:1788572980.419941:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419942:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132cd90 page ffff9c373132cd80 at index 10. 00000008:00000001:0.0:1788572980.419943:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419945:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419946:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec845344c0 index 10 00000020:00000001:0.0:1788572980.419947:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419947:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419949:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419949:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419950:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419951:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419951:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419952:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419953:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419954:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419955:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419955:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419957:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000e558af4b[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 11] 00000020:00000001:0.0:1788572980.419958:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419959:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419959:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419960:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419961:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419964:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419965:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419965:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419966:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419968:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000e558af4b[2 000000006e2ad75d 2 1 0000000000000000 0 1 11] 00000020:00000001:0.0:1788572980.419968:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419969:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419970:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.419971:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132cae0 00000008:00000001:0.0:1788572980.419972:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.419973:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132caf0 page ffff9c373132cae0 at index 11. 00000008:00000001:0.0:1788572980.419974:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.419975:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.419976:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84a3ab00 index 11 00000020:00000001:0.0:1788572980.419977:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419978:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.419979:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.419980:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419980:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419981:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.419982:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.419983:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.419983:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.419984:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.419985:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.419986:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419987:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000105c0286[2 000000006e2ad75d 1 1 000000003e6fbc71 0 1 12] 00000020:00000001:0.0:1788572980.419988:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.419989:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.419989:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.419990:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.419991:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419992:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.419992:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.419993:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.419994:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.419995:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000105c0286[2 000000006e2ad75d 2 1 0000000000000000 0 1 12] 00000020:00000001:0.0:1788572980.419996:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.419997:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.419997:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420001:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c373132cf40 00000008:00000001:0.0:1788572980.420001:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420002:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c373132cf50 page ffff9c373132cf40 at index 12. 00000008:00000001:0.0:1788572980.420003:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420005:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420006:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84966dc0 index 12 00000020:00000001:0.0:1788572980.420007:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420008:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420009:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420010:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420010:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420011:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420012:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420013:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420013:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420014:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420015:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420016:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420017:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@0000000007476714[2 000000006e2ad75d 1 1 000000003e6fbc71 0 0 13] 00000020:00000001:0.0:1788572980.420018:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420019:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420020:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420020:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420021:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420022:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420023:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420023:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420024:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420025:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@0000000007476714[2 000000006e2ad75d 2 1 0000000000000000 0 0 13] 00000020:00000001:0.0:1788572980.420026:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420027:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420028:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420028:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c3731fe1840 00000008:00000001:0.0:1788572980.420029:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420030:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c3731fe1850 page ffff9c3731fe1840 at index 13. 00000008:00000001:0.0:1788572980.420031:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420034:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420035:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8455b0c0 index 13 00000020:00000001:0.0:1788572980.420036:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420040:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420043:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420043:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420044:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420045:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420046:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420048:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000c87c6eca[1 000000006e2ad75d 5 1 0000000000000000 0 0 0] 00000020:00000001:0.0:1788572980.420049:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420051:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420054:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace620. 00000020:00000001:0.0:1788572980.420057:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420058:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420059:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420060:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000d94249a2[1 000000006e2ad75d 5 1 0000000000000000 0 1 1] 00000020:00000001:0.0:1788572980.420061:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420062:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420063:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eaceb60. 00000020:00000001:0.0:1788572980.420065:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420065:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420066:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420067:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000000dc93ebd[1 000000006e2ad75d 5 1 0000000000000000 0 1 2] 00000020:00000001:0.0:1788572980.420068:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420069:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420070:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace700. 00000020:00000001:0.0:1788572980.420071:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420071:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420072:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420073:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000eaa488d2[1 000000006e2ad75d 5 1 0000000000000000 0 1 3] 00000020:00000001:0.0:1788572980.420074:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420075:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420076:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace9a0. 00000020:00000001:0.0:1788572980.420077:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420077:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420078:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420079:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000af683df1[1 000000006e2ad75d 5 1 0000000000000000 0 1 4] 00000020:00000001:0.0:1788572980.420080:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420081:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420082:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132c620. 00000020:00000001:0.0:1788572980.420084:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420084:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420085:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420105:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000005b41729e[1 000000006e2ad75d 5 1 0000000000000000 0 1 5] 00000020:00000001:0.0:1788572980.420106:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420108:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420110:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132cb60. 00000020:00000001:0.0:1788572980.420112:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420113:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420115:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420117:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000001b093ab6[1 000000006e2ad75d 5 1 0000000000000000 0 1 6] 00000020:00000001:0.0:1788572980.420119:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420120:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420122:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132c9a0. 00000020:00000001:0.0:1788572980.420124:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420125:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420126:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420129:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000534527b3[1 000000006e2ad75d 5 1 0000000000000000 0 1 7] 00000020:00000001:0.0:1788572980.420130:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420132:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420133:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132c700. 00000020:00000001:0.0:1788572980.420135:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420136:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420137:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420140:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b4e56d35[1 000000006e2ad75d 5 1 0000000000000000 0 1 8] 00000020:00000001:0.0:1788572980.420142:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420143:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420144:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132c1c0. 00000020:00000001:0.0:1788572980.420146:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420146:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420147:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420149:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000d6fcde9e[1 000000006e2ad75d 5 1 0000000000000000 0 1 9] 00000020:00000001:0.0:1788572980.420150:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420151:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420152:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132c2a0. 00000020:00000001:0.0:1788572980.420153:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420153:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420154:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420155:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b05f2f1f[1 000000006e2ad75d 5 1 0000000000000000 0 1 10] 00000020:00000001:0.0:1788572980.420156:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420157:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420158:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132cd20. 00000020:00000001:0.0:1788572980.420159:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420159:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420164:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420166:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000e558af4b[1 000000006e2ad75d 5 1 0000000000000000 0 1 11] 00000020:00000001:0.0:1788572980.420167:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420168:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420169:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132ca80. 00000020:00000001:0.0:1788572980.420171:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420172:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420172:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420174:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000105c0286[1 000000006e2ad75d 5 1 0000000000000000 0 1 12] 00000020:00000001:0.0:1788572980.420174:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420175:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420176:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c373132cee0. 00000020:00000001:0.0:1788572980.420177:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420178:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420178:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420180:0:243062:0:(cl_page.c:556:cl_batch_put()) page@0000000007476714[1 000000006e2ad75d 5 1 0000000000000000 0 0 13] 00000020:00000001:0.0:1788572980.420181:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420181:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420182:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c3731fe17e0. 00000020:00000001:0.0:1788572980.420183:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420184:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000008:00000001:0.0:1788572980.420221:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.420222:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.420224:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.420224:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.420225:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.420226:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xc88:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.420228:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.420229:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.420230:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.420231:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.420232:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420234:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8700 -> ffff9c3720b2e148 00010000:00000001:0.0:1788572980.420235:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.420236:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420237:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8700 -> ffff9c3720b2e148 00000020:00000001:0.0:1788572980.420239:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.420240:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 53248to 0 00020000:00000001:0.0:1788572980.420241:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.420243:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420246:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420248:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8700 -> ffff9c3720b2e148 00000020:00000001:0.0:1788572980.420249:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3720b2e148 -> ffff9c3720b2e148 00000020:00000001:0.0:1788572980.420250:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.420251:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420252:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c371f2c8700 -> ffff9c3720b2e148 00000008:00000001:0.0:1788572980.420254:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.420255:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.420256:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.420258:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.420259:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.420260:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.420261:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.420263:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.420265:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372007f000 with handle 0x1441586093165185 from hash 00010000:00000001:0.0:1788572980.420266:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.420267:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.420268:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.420269:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.420270:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.420271:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.420272:0:243062:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788572980.420274:0:243062:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9b588a7/0x1441586093165177 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x525a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0xd9894672d9992461 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00000001:0.0:1788572980.420279:0:243062:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788572980.420280:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.420281:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359186941352 : -109714522610264 : ffff9c3718da65a8) 00000020:00001000:0.0:1788572980.420283:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.420284:0:243062:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788572980.420285:0:243062:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788572980.420286:0:243062:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572980.420287:0:243062:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446634359065793240 : -109714643758376 : ffff9c3711a1d2d8) 00000020:00001000:0.0:1788572980.420288:0:243062:0:(cl_object.c:823:cl_env_get()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.420289:0:243062:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788572980.420290:0:243062:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788572980.420295:0:243062:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.420296:0:243062:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420297:0:243062:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788572980.420299:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3705331180 -> ffff9c3711590d00 00000020:00000001:0.0:1788572980.420300:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3711590d00 -> ffff9c3711590d00 00000020:00000001:0.0:1788572980.420301:0:243062:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572980.420302:0:243062:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572980.420303:0:243062:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0xc85:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572980.420305:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.420305:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572980.420306:0:243062:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572980.420308:0:243062:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0xc85:0x0]io ffff9c3730ce2d60 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788572980.420309:0:243062:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572980.420310:0:243062:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572980.420312:0:243062:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572980.420313:0:243062:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572980.420314:0:243062:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420314:0:243062:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572980.420315:0:243062:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572980.420316:0:243062:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788572980.420316:0:243062:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420318:0:243062:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000020:00000001:0.0:1788572980.420320:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420320:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420321:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420322:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420323:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420323:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420324:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420325:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420325:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420326:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420327:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420328:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420329:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420329:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420330:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420331:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420331:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420332:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420333:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420336:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420337:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420337:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420338:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420339:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420340:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420341:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000020:00000001:0.0:1788572980.420341:0:243062:0:(cl_page.c:534:cl_page_get()) Process entered 00000020:00000001:0.0:1788572980.420342:0:243062:0:(cl_page.c:536:cl_page_get()) Process leaving 00000008:00000001:0.0:1788572980.420343:0:243062:0:(osc_lock.c:1343:osc_obj_dlmlock_at_pgoff()) Process entered 00000008:00000001:0.0:1788572980.420344:0:243062:0:(osc_request.c:3331:osc_match_base()) Process entered 00010000:00000001:0.0:1788572980.420345:0:243062:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788572980.420347:0:243062:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff9c37421e3000 count: 2 00010000:00000001:0.0:1788572980.420349:0:243062:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788572980.420351:0:243062:0:(osc_request.c:3346:osc_match_base()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420352:0:243062:0:(osc_lock.c:1377:osc_obj_dlmlock_at_pgoff()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420353:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420354:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420355:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420356:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420357:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420358:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420360:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000a97a3a6[2 00000000786c64c7 1 1 000000003e6fbc71 0 0 0] 00000020:00000001:0.0:1788572980.420361:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420362:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420363:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420364:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420365:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420366:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420366:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420367:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420368:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420369:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000a97a3a6[2 00000000786c64c7 2 1 0000000000000000 0 0 0] 00000020:00000001:0.0:1788572980.420370:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420371:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420371:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420372:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eaced80 00000008:00000001:0.0:1788572980.420373:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420374:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eaced90 page ffff9c371eaced80 at index 0. 00000008:00000001:0.0:1788572980.420375:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420381:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420383:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec842efec0 index 0 00000020:00000001:0.0:1788572980.420384:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420385:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420388:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420389:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420390:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420391:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420391:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420393:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420393:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420395:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420395:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420396:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420398:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b7f20d00[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 1] 00000020:00000001:0.0:1788572980.420398:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420399:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420400:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420401:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420402:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420402:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420403:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420403:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420404:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420406:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b7f20d00[2 00000000786c64c7 2 1 0000000000000000 0 1 1] 00000020:00000001:0.0:1788572980.420406:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420407:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420408:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420408:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eaceae0 00000008:00000001:0.0:1788572980.420409:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420410:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eaceaf0 page ffff9c371eaceae0 at index 1. 00000008:00000001:0.0:1788572980.420411:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420412:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420413:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84767040 index 1 00000020:00000001:0.0:1788572980.420414:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420415:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420416:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420416:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420417:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420418:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420421:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420422:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420423:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420424:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420425:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420426:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420428:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000af69efc[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 2] 00000020:00000001:0.0:1788572980.420428:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420429:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420430:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420431:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420432:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420432:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420433:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420433:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420434:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420436:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000000af69efc[2 00000000786c64c7 2 1 0000000000000000 0 1 2] 00000020:00000001:0.0:1788572980.420436:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420437:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420438:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420438:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace220 00000008:00000001:0.0:1788572980.420439:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420440:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace230 page ffff9c371eace220 at index 2. 00000008:00000001:0.0:1788572980.420441:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420443:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420443:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84953940 index 2 00000020:00000001:0.0:1788572980.420444:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420445:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420446:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420447:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420448:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420448:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420449:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420450:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420450:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420451:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420452:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420453:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420454:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000869848e4[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 3] 00000020:00000001:0.0:1788572980.420455:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420459:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420459:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420460:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420461:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420462:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420462:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420463:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420464:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420466:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000869848e4[2 00000000786c64c7 2 1 0000000000000000 0 1 3] 00000020:00000001:0.0:1788572980.420466:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420467:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420468:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420468:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace300 00000008:00000001:0.0:1788572980.420469:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420470:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace310 page ffff9c371eace300 at index 3. 00000008:00000001:0.0:1788572980.420471:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420473:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420473:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84313500 index 3 00000020:00000001:0.0:1788572980.420474:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420475:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420476:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420477:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420478:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420478:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420479:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420480:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420481:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420482:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420482:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420483:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420485:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b94bbfca[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 4] 00000020:00000001:0.0:1788572980.420485:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420486:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420487:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420487:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420488:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420489:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420490:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420490:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420491:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420493:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b94bbfca[2 00000000786c64c7 2 1 0000000000000000 0 1 4] 00000020:00000001:0.0:1788572980.420496:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420497:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420497:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420498:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eacef40 00000008:00000001:0.0:1788572980.420499:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420500:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eacef50 page ffff9c371eacef40 at index 4. 00000008:00000001:0.0:1788572980.420501:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420502:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420503:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8471ef00 index 4 00000020:00000001:0.0:1788572980.420520:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420521:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420523:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420524:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420524:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420525:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420526:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420527:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420527:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420529:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420529:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420530:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420532:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000008817b5cb[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 5] 00000020:00000001:0.0:1788572980.420533:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420534:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420535:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420536:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420537:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420539:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420540:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420541:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420542:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420545:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000008817b5cb[2 00000000786c64c7 2 1 0000000000000000 0 1 5] 00000020:00000001:0.0:1788572980.420547:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420548:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420549:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420550:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace3e0 00000008:00000001:0.0:1788572980.420552:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420553:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace3f0 page ffff9c371eace3e0 at index 5. 00000008:00000001:0.0:1788572980.420556:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420564:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420566:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8476bb40 index 5 00000020:00000001:0.0:1788572980.420569:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420570:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420573:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420574:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420575:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420576:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420578:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420579:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420581:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420583:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420584:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420586:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420589:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000bbc21d82[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 6] 00000020:00000001:0.0:1788572980.420591:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420593:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420594:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420595:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420597:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420598:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420599:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420600:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420601:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420604:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000bbc21d82[2 00000000786c64c7 2 1 0000000000000000 0 1 6] 00000020:00000001:0.0:1788572980.420606:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420607:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420607:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420608:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace5a0 00000008:00000001:0.0:1788572980.420609:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420610:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace5b0 page ffff9c371eace5a0 at index 6. 00000008:00000001:0.0:1788572980.420611:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420613:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420614:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec848d2200 index 6 00000020:00000001:0.0:1788572980.420615:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420616:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420617:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420618:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420619:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420620:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420625:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420626:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420627:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420628:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420629:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420630:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420632:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000470ea026[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 7] 00000020:00000001:0.0:1788572980.420633:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420659:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420660:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420661:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420663:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420664:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420665:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420665:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420666:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420668:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000470ea026[2 00000000786c64c7 2 1 0000000000000000 0 1 7] 00000020:00000001:0.0:1788572980.420669:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420670:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420670:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420672:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace840 00000008:00000001:0.0:1788572980.420673:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420674:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace850 page ffff9c371eace840 at index 7. 00000008:00000001:0.0:1788572980.420676:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420678:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420679:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84a98280 index 7 00000020:00000001:0.0:1788572980.420682:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420683:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420685:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420686:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420687:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420688:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420689:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420691:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420692:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420693:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420694:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420695:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420697:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000a4c09dae[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 8] 00000020:00000001:0.0:1788572980.420698:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420703:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420705:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420706:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420707:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420708:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420709:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420710:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420710:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420712:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000a4c09dae[2 00000000786c64c7 2 1 0000000000000000 0 1 8] 00000020:00000001:0.0:1788572980.420713:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420714:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420714:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420715:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace060 00000008:00000001:0.0:1788572980.420716:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420717:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace070 page ffff9c371eace060 at index 8. 00000008:00000001:0.0:1788572980.420718:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420719:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420720:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84894dc0 index 8 00000020:00000001:0.0:1788572980.420721:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420722:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420723:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420724:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420724:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420725:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420726:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420727:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420727:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420728:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420729:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420730:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420731:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b59b5624[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 9] 00000020:00000001:0.0:1788572980.420732:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420733:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420733:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420734:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420735:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420736:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420736:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420737:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420738:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420739:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b59b5624[2 00000000786c64c7 2 1 0000000000000000 0 1 9] 00000020:00000001:0.0:1788572980.420744:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420745:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420746:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420747:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace140 00000008:00000001:0.0:1788572980.420748:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420749:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace150 page ffff9c371eace140 at index 9. 00000008:00000001:0.0:1788572980.420750:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420751:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420752:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84894d40 index 9 00000020:00000001:0.0:1788572980.420753:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420754:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420755:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420756:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420757:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420757:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420758:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420759:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420760:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420761:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420761:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420762:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420764:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000c0d2f896[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 10] 00000020:00000001:0.0:1788572980.420764:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420765:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420766:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420767:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420768:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420768:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420769:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420769:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420770:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420772:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000c0d2f896[2 00000000786c64c7 2 1 0000000000000000 0 1 10] 00000020:00000001:0.0:1788572980.420772:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420773:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420774:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420774:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace920 00000008:00000001:0.0:1788572980.420775:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420776:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace930 page ffff9c371eace920 at index 10. 00000008:00000001:0.0:1788572980.420777:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420781:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420782:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec8451c000 index 10 00000020:00000001:0.0:1788572980.420784:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420785:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420786:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420787:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420788:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420788:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420789:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420790:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420791:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420792:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420792:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420793:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420795:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000007ca6d0ad[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 11] 00000020:00000001:0.0:1788572980.420796:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420796:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420797:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420798:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420799:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420799:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420800:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420801:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420801:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420803:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000007ca6d0ad[2 00000000786c64c7 2 1 0000000000000000 0 1 11] 00000020:00000001:0.0:1788572980.420804:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420804:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420805:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420806:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eacee60 00000008:00000001:0.0:1788572980.420806:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420807:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eacee70 page ffff9c371eacee60 at index 11. 00000008:00000001:0.0:1788572980.420808:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420809:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420810:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec842ab9c0 index 11 00000020:00000001:0.0:1788572980.420811:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420812:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420813:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420814:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420814:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420815:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420818:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420819:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420820:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420822:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420822:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420823:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420825:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000001ede14c4[2 00000000786c64c7 1 1 000000003e6fbc71 0 1 12] 00000020:00000001:0.0:1788572980.420826:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420826:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420827:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420828:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420829:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420829:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420830:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420831:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420831:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420833:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@000000001ede14c4[2 00000000786c64c7 2 1 0000000000000000 0 1 12] 00000020:00000001:0.0:1788572980.420834:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420834:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420835:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420836:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eaceca0 00000008:00000001:0.0:1788572980.420837:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420837:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eacecb0 page ffff9c371eaceca0 at index 12. 00000008:00000001:0.0:1788572980.420838:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420840:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420841:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec84534e40 index 12 00000020:00000001:0.0:1788572980.420842:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420842:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420844:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420844:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420845:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420846:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420846:0:243062:0:(cl_page.c:700:__cl_page_own()) Process entered 00000020:00000001:0.0:1788572980.420847:0:243062:0:(cl_io.c:1145:cl_io_top()) Process entered 00000020:00000001:0.0:1788572980.420848:0:243062:0:(cl_io.c:1148:cl_io_top()) Process leaving (rc=18446634359588793696 : -109714120757920 : ffff9c3730ce2d60) 00000020:00000001:0.0:1788572980.420849:0:243062:0:(cl_page.c:634:cl_page_owner_set()) Process entered 00000020:00000001:0.0:1788572980.420849:0:243062:0:(cl_page.c:637:cl_page_owner_set()) Process leaving 00000020:00000001:0.0:1788572980.420850:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420852:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b8805fc1[2 00000000786c64c7 1 1 000000003e6fbc71 0 0 13] 00000020:00000001:0.0:1788572980.420852:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 1 -> 2 00000020:00000001:0.0:1788572980.420856:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000020:00000040:0.0:1788572980.420856:0:243062:0:(cl_page.c:744:__cl_page_own()) res 0 00000020:00000001:0.0:1788572980.420857:0:243062:0:(cl_page.c:747:__cl_page_own()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.420858:0:243062:0:(cl_page.c:976:cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420859:0:243062:0:(cl_page.c:928:__cl_page_delete()) Process entered 00000020:00000001:0.0:1788572980.420859:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420860:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420861:0:243062:0:(cl_page.c:504:__cl_page_state_set()) Process entered 00000020:00000001:0.0:1788572980.420862:0:243062:0:(cl_page.c:507:__cl_page_state_set()) page@00000000b8805fc1[2 00000000786c64c7 2 1 0000000000000000 0 0 13] 00000020:00000001:0.0:1788572980.420863:0:243062:0:(cl_page.c:507:__cl_page_state_set()) 2 -> 5 00000020:00000001:0.0:1788572980.420864:0:243062:0:(cl_page.c:515:__cl_page_state_set()) Process leaving 00000008:00000001:0.0:1788572980.420865:0:243062:0:(osc_page.c:137:osc_page_delete()) Process entered 00000008:00000001:0.0:1788572980.420865:0:243062:0:(osc_page.c:138:osc_page_delete()) ffff9c371eace4c0 00000008:00000001:0.0:1788572980.420866:0:243062:0:(osc_cache.c:2730:osc_teardown_async_page()) Process entered 00000008:00000040:0.0:1788572980.420867:0:243062:0:(osc_cache.c:2733:osc_teardown_async_page()) teardown oap ffff9c371eace4d0 page ffff9c371eace4c0 at index 13. 00000008:00000001:0.0:1788572980.420868:0:243062:0:(osc_cache.c:2755:osc_teardown_async_page()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572980.420871:0:243062:0:(osc_page.c:169:osc_page_delete()) Process leaving 00000080:00000020:0.0:1788572980.420872:0:243062:0:(vvp_page.c:56:vvp_page_delete()) delete page ffffeeec849b66c0 index 13 00000020:00000001:0.0:1788572980.420873:0:243062:0:(cl_page.c:946:__cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420874:0:243062:0:(cl_page.c:978:cl_page_delete()) Process leaving 00000020:00000001:0.0:1788572980.420876:0:243062:0:(cl_page.c:645:__cl_page_disown()) Process entered 00000020:00000001:0.0:1788572980.420877:0:243062:0:(cl_page.c:623:cl_page_owner_clear()) Process entered 00000020:00000001:0.0:1788572980.420878:0:243062:0:(cl_page.c:629:cl_page_owner_clear()) Process leaving 00000020:00000001:0.0:1788572980.420878:0:243062:0:(cl_page.c:660:__cl_page_disown()) Process leaving 00000020:00000001:0.0:1788572980.420879:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420881:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000000a97a3a6[1 00000000786c64c7 5 1 0000000000000000 0 0 0] 00000020:00000001:0.0:1788572980.420881:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420882:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420884:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eaced20. 00000020:00000001:0.0:1788572980.420886:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420886:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420887:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420888:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b7f20d00[1 00000000786c64c7 5 1 0000000000000000 0 1 1] 00000020:00000001:0.0:1788572980.420890:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420890:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420891:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eacea80. 00000020:00000001:0.0:1788572980.420893:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420893:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420894:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420895:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000000af69efc[1 00000000786c64c7 5 1 0000000000000000 0 1 2] 00000020:00000001:0.0:1788572980.420896:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420899:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420901:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace1c0. 00000020:00000001:0.0:1788572980.420902:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420903:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420904:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420906:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000869848e4[1 00000000786c64c7 5 1 0000000000000000 0 1 3] 00000020:00000001:0.0:1788572980.420906:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420907:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420908:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace2a0. 00000020:00000001:0.0:1788572980.420909:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420910:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420910:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420912:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b94bbfca[1 00000000786c64c7 5 1 0000000000000000 0 1 4] 00000020:00000001:0.0:1788572980.420912:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420913:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420914:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eaceee0. 00000020:00000001:0.0:1788572980.420915:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420916:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420916:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420918:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000008817b5cb[1 00000000786c64c7 5 1 0000000000000000 0 1 5] 00000020:00000001:0.0:1788572980.420918:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420919:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420920:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace380. 00000020:00000001:0.0:1788572980.420921:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420922:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420922:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420924:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000bbc21d82[1 00000000786c64c7 5 1 0000000000000000 0 1 6] 00000020:00000001:0.0:1788572980.420924:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420925:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420926:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace540. 00000020:00000001:0.0:1788572980.420927:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420928:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420928:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420930:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000470ea026[1 00000000786c64c7 5 1 0000000000000000 0 1 7] 00000020:00000001:0.0:1788572980.420930:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420931:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420932:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace7e0. 00000020:00000001:0.0:1788572980.420933:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420933:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420934:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420936:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000a4c09dae[1 00000000786c64c7 5 1 0000000000000000 0 1 8] 00000020:00000001:0.0:1788572980.420939:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420940:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420941:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace000. 00000020:00000001:0.0:1788572980.420942:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420943:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420944:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420946:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b59b5624[1 00000000786c64c7 5 1 0000000000000000 0 1 9] 00000020:00000001:0.0:1788572980.420946:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420947:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420948:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace0e0. 00000020:00000001:0.0:1788572980.420949:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420949:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420950:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420951:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000c0d2f896[1 00000000786c64c7 5 1 0000000000000000 0 1 10] 00000020:00000001:0.0:1788572980.420952:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420953:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420954:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace8c0. 00000020:00000001:0.0:1788572980.420955:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420955:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420956:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420957:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000007ca6d0ad[1 00000000786c64c7 5 1 0000000000000000 0 1 11] 00000020:00000001:0.0:1788572980.420958:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420959:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420960:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eacee00. 00000020:00000001:0.0:1788572980.420961:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420961:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420962:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420963:0:243062:0:(cl_page.c:556:cl_batch_put()) page@000000001ede14c4[1 00000000786c64c7 5 1 0000000000000000 0 1 12] 00000020:00000001:0.0:1788572980.420964:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420965:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420966:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eacec40. 00000020:00000001:0.0:1788572980.420967:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420967:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000020:00000001:0.0:1788572980.420968:0:243062:0:(cl_page.c:554:cl_batch_put()) Process entered 00000020:00000001:0.0:1788572980.420969:0:243062:0:(cl_page.c:556:cl_batch_put()) page@00000000b8805fc1[1 00000000786c64c7 5 1 0000000000000000 0 0 13] 00000020:00000001:0.0:1788572980.420970:0:243062:0:(cl_page.c:556:cl_batch_put()) 1 00000020:00000001:0.0:1788572980.420971:0:243062:0:(cl_page.c:236:cl_page_free()) Process entered 00000020:00000010:0.0:1788572980.420972:0:243062:0:(cl_page.c:223:__cl_page_free()) slab-freed 'cl_page': 224 at ffff9c371eace460. 00000020:00000001:0.0:1788572980.420973:0:243062:0:(cl_page.c:274:cl_page_free()) Process leaving 00000020:00000001:0.0:1788572980.420973:0:243062:0:(cl_page.c:569:cl_batch_put()) Process leaving 00000008:00000001:0.0:1788572980.421001:0:243062:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572980.421005:0:243062:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572980.421006:0:243062:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572980.421007:0:243062:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572980.421008:0:243062:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572980.421009:0:243062:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0xc85:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788572980.421011:0:243062:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572980.421012:0:243062:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788572980.421013:0:243062:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.421014:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3711a1d2d0 00000008:00000001:0.0:1788572980.421016:0:243062:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421017:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3705331180 -> ffff9c3711590d00 00010000:00000001:0.0:1788572980.421018:0:243062:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788572980.421020:0:243062:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421021:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3705331180 -> ffff9c3711590d00 00000020:00000001:0.0:1788572980.421022:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788572980.421023:0:243062:0:(osc_object.c:161:osc_attr_update()) set kms from 53248to 0 00020000:00000001:0.0:1788572980.421025:0:243062:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788572980.421026:0:243062:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421027:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421028:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3705331180 -> ffff9c3711590d00 00000020:00000001:0.0:1788572980.421029:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3711590d00 -> ffff9c3711590d00 00000020:00000001:0.0:1788572980.421030:0:243062:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788572980.421031:0:243062:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421033:0:243062:0:(cl_object.c:143:cl_object_top()) ffff9c3705331180 -> ffff9c3711590d00 00000008:00000001:0.0:1788572980.421034:0:243062:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788572980.421035:0:243062:0:(cl_object.c:917:cl_env_put()) 1@ffff9c3718da65a0 00000008:00000001:0.0:1788572980.421036:0:243062:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.421038:0:243062:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788572980.421039:0:243062:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788572980.421040:0:243062:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788572980.421041:0:243062:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788572980.421043:0:243062:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788572980.421045:0:243062:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372007f600 with handle 0x1441586093165177 from hash 00010000:00000001:0.0:1788572980.421046:0:243062:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788572980.421047:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788572980.421048:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421049:0:243062:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788572980.421050:0:243062:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788572980.421055:0:243062:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788572980.421057:0:243062:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=16 : 16 : 10) 00010000:00000001:0.0:1788572980.421059:0:243062:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788572980.421061:0:243062:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788572980.421067:0:243062:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5c00. 00000020:00000040:0.0:1788572980.421070:0:243062:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000100:00000001:0.0:1788572980.421075:0:243062:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572980.421077:0:243062:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572980.421081:0:243062:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572980.421088:0:243062:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de6200. 00000100:00000001:0.0:1788572980.421095:0:243062:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788572980.421098:0:243062:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788572980.421104:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000545753e5/0x1441586093168476 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5269:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2244 expref: -99 pid: 261328 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421114:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001ab58db4/0x144158609316845a lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5268:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a21fe expref: -99 pid: 261324 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421123:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000f9053d9f/0x144158609316843e lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5267:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a21b8 expref: -99 pid: 261320 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421132:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000aa77ba1e/0x1441586093168422 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5266:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2172 expref: -99 pid: 261316 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421140:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000004a018978/0x1441586093168406 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5265:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a212c expref: -99 pid: 261312 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421147:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001b7036a2/0x14415860931683ea lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5264:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a20e6 expref: -99 pid: 261308 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421160:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000043229ee0/0x14415860931683ce lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5263:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a20a0 expref: -99 pid: 261304 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421165:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000012528090/0x14415860931683b2 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5262:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a205a expref: -99 pid: 261300 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421169:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000006555ea31/0x1441586093168396 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5261:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2014 expref: -99 pid: 261296 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421174:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 1/0,0 mode: --/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421178:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000573e3650/0x14415860931682ee lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1dc8 expref: -99 pid: 261286 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421182:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000fed4281f/0x14415860931682d2 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1d82 expref: -99 pid: 261282 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421187:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000002e1c4230/0x14415860931682af lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1c40 expref: -99 pid: 261276 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421191:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788572980.421196:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000e69e32cb/0x1441586093165185 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d9992484 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788572980.421204:0:243062:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9b588a7/0x1441586093165177 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x525a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d9992461 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788572980.421208:0:243062:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 16 locks packed 00010000:00000001:0.0:1788572980.421209:0:243062:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=16 : 16 : 10) 00000100:00000001:0.0:1788572980.421214:0:243062:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572980.421217:0:243062:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572980.421227:0:243062:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000543ab7fa] to pc [ptlrpcd_00_00+0] req@ffff9c372d6b5c00 x1875440810045184/t0(0) o103->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 440/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788572980.421246:0:243062:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788572980.421248:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421250:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000545753e5/0x1441586093168476 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5269:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2244 expref: -99 pid: 261328 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421256:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83580. 00010000:00000040:0.0:1788572980.421260:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c371fc25600 count: 0 00010000:00000010:0.0:1788572980.421264:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c371fc25000. 00010000:00000001:0.0:1788572980.421270:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421272:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421274:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001ab58db4/0x144158609316845a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5268:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a21fe expref: -99 pid: 261324 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421279:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372eccb980. 00010000:00000040:0.0:1788572980.421281:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713ffec00 count: 0 00010000:00000010:0.0:1788572980.421283:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3713ffed00. 00010000:00000001:0.0:1788572980.421295:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421297:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421299:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000f9053d9f/0x144158609316843e lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5267:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a21b8 expref: -99 pid: 261320 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421322:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372d091f00. 00010000:00000040:0.0:1788572980.421326:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3708c76200 count: 0 00010000:00000010:0.0:1788572980.421330:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3708c76b00. 00010000:00000001:0.0:1788572980.421335:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421337:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421341:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000aa77ba1e/0x1441586093168422 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5266:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2172 expref: -99 pid: 261316 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421348:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985ac0. 00010000:00000040:0.0:1788572980.421351:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000010:0.0:1788572980.421354:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3720692c00. 00010000:00000001:0.0:1788572980.421359:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421361:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421366:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000004a018978/0x1441586093168406 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5265:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a212c expref: -99 pid: 261312 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421374:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985040. 00010000:00000040:0.0:1788572980.421378:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692b00 count: 0 00010000:00000010:0.0:1788572980.421381:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3720692a00. 00010000:00000001:0.0:1788572980.421384:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421385:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421386:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000001b7036a2/0x14415860931683ea lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5264:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a20e6 expref: -99 pid: 261308 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421393:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985e00. 00010000:00000040:0.0:1788572980.421397:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000010:0.0:1788572980.421401:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3720692200. 00010000:00000001:0.0:1788572980.421407:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421414:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421416:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000043229ee0/0x14415860931683ce lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5263:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a20a0 expref: -99 pid: 261304 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421428:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985b40. 00010000:00000040:0.0:1788572980.421432:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692800 count: 0 00010000:00000010:0.0:1788572980.421438:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3720692700. 00010000:00000001:0.0:1788572980.421444:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421448:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421450:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 0000000012528090/0x14415860931683b2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5262:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a205a expref: -99 pid: 261300 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421461:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985d40. 00010000:00000040:0.0:1788572980.421463:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3707e8f300 count: 0 00010000:00000010:0.0:1788572980.421467:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3720692400. 00010000:00000001:0.0:1788572980.421470:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421471:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421474:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000006555ea31/0x1441586093168396 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5261:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a2014 expref: -99 pid: 261296 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421482:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985c00. 00010000:00000040:0.0:1788572980.421485:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3707e8fb00 count: 0 00010000:00000010:0.0:1788572980.421487:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3707e8fd00. 00010000:00000001:0.0:1788572980.421490:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421491:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421493:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9d393ce/0x144158609316830a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x5260:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1e0e expref: -99 pid: 261290 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421498:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c838c0. 00010000:00000040:0.0:1788572980.421499:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37050f7a00 count: 0 00010000:00000010:0.0:1788572980.421501:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37050f7100. 00010000:00000001:0.0:1788572980.421519:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421520:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421522:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000573e3650/0x14415860931682ee lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1dc8 expref: -99 pid: 261286 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421527:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3707985200. 00010000:00000040:0.0:1788572980.421529:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3707e8f500 count: 0 00010000:00000010:0.0:1788572980.421531:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c3707e8f400. 00010000:00000001:0.0:1788572980.421534:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421534:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421536:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000fed4281f/0x14415860931682d2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1d82 expref: -99 pid: 261282 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421540:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83540. 00010000:00000040:0.0:1788572980.421542:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37050f7600 count: 0 00010000:00000010:0.0:1788572980.421544:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37050f7700. 00010000:00000001:0.0:1788572980.421546:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421547:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421548:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000002e1c4230/0x14415860931682af lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1c40 expref: -99 pid: 261276 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421552:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83b00. 00010000:00000040:0.0:1788572980.421553:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37050f7800 count: 0 00010000:00000010:0.0:1788572980.421555:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37050f7000. 00010000:00000001:0.0:1788572980.421558:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421558:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421560:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 000000005029e6ea/0x144158609316825b lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d99a1bfa expref: -99 pid: 261272 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788572980.421564:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c3713c83000. 00010000:00000040:0.0:1788572980.421565:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37050f7300 count: 0 00010000:00000010:0.0:1788572980.421570:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37050f7c00. 00010000:00000001:0.0:1788572980.421575:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421576:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421577:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000e69e32cb/0x1441586093165185 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d9992484 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00000010:0.0:1788572980.421582:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372d06a080. 00010000:00000040:0.0:1788572980.421583:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37421e3600 count: 0 00010000:00000010:0.0:1788572980.421585:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37421e3100. 00010000:00000001:0.0:1788572980.421588:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421589:0:243062:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788572980.421590:0:243062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9c3711155800 lock: 00000000c9b588a7/0x1441586093165177 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x525a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0xd9894672d9992461 expref: -99 pid: 258785 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00000010:0.0:1788572980.421595:0:243062:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff9c372d06a140. 00010000:00000040:0.0:1788572980.421596:0:243062:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c37421e3000 count: 0 00010000:00000010:0.0:1788572980.421598:0:243062:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff9c37421e3b00. 00010000:00000001:0.0:1788572980.421600:0:243062:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788572980.421601:0:243062:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.0:1788572980.421603:0:243062:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff9c37380b6240. 00010000:00000001:0.0:1788572980.421605:0:243062:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.421607:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572980.421608:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572980.421609:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572980.421610:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572980.421611:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572980.421612:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572980.421613:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572980.421614:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572980.421615:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572980.801741:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308119775, at 4308119808 00000100:00000010:0.0:1788572980.801757:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788572983.042430:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572983.042444:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572983.042446:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572983.042449:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572983.042453:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572983.042457:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13453 00000400:00000200:0.0:1788572983.042465:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572983.042472:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572983.042475:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572983.042479:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042482:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042485:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042487:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572983.042491:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572983.042493:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572983.042523:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572983.042526:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572983.042529:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572983.042531:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572983.042535:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042544:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042546:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042548:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572983.042555:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572983.042559:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042562:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572983.042564:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572983.042567:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572983.042570:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042572:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572983.042574:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572983.042579:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788572983.042588:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572983.042591:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572983.042593:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042595:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572983.042597:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572983.042600:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572983.042604:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788572983.042609:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.042614:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810046080/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572983.042621:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572983.042623:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.042627:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810046080/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572983.042667:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572983.042670:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572983.042673:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572983.042675:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572983.042676:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572983.042678:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572983.042681:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13453 00000400:00000200:0.0:1788572983.042684:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572983.042688:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572983.042690:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572983.042692:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042694:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042697:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042699:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572983.042701:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572983.042703:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572983.042704:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572983.042711:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572983.042715:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572983.042718:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572983.042722:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042725:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042728:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042731:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572983.042739:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572983.042745:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042752:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572983.042758:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572983.042763:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572983.042767:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042771:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572983.042775:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572983.042777:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042779:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572983.042783:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788572983.042787:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572983.042791:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572983.042794:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042796:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572983.042798:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572983.042801:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572983.042806:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788572983.042811:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.042818:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810046208/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572983.042828:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572983.042832:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.042837:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810046208/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572983.042857:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572983.042863:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572983.042869:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572983.042870:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572983.042872:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572983.042875:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572983.042880:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13453 00000400:00000200:0.0:1788572983.042887:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572983.042893:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572983.042896:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572983.042900:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042903:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042907:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042910:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572983.042913:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572983.042914:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572983.042916:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572983.042917:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572983.042922:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572983.042924:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572983.042927:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042930:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.042932:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042933:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572983.042935:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572983.042939:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572983.042944:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572983.042949:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572983.042953:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572983.042957:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042958:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572983.042960:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572983.042963:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042968:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572983.042971:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788572983.042975:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572983.042979:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572983.042982:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.042984:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572983.042986:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572983.042989:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572983.042994:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788572983.042998:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.043004:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810046336/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572983.043017:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572983.043020:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.043027:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810046336/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572983.043044:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572983.043047:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572983.043052:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572983.043052:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572983.043054:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572983.043056:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572983.043059:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13453 00000400:00000200:0.0:1788572983.043063:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572983.043067:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572983.043068:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572983.043081:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043085:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.043089:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043094:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572983.043096:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572983.043099:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572983.043101:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572983.043104:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572983.043108:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572983.043111:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572983.043118:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043123:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.043130:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043136:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572983.043143:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572983.043150:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043156:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572983.043161:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572983.043166:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572983.043173:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043178:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572983.043182:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572983.043186:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043189:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572983.043196:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788572983.043202:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572983.043207:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572983.043208:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043209:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572983.043211:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572983.043215:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572983.043220:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788572983.043234:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.043242:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810046464/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572983.043253:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572983.043256:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.043260:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810046464/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572983.043271:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572983.043273:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572983.043275:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572983.043276:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572983.043277:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572983.043278:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572983.043280:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13453 00000400:00000200:0.0:1788572983.043288:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572983.043292:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572983.043294:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572983.043299:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043302:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.043306:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043310:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572983.043314:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572983.043316:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572983.043319:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572983.043322:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572983.043325:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572983.043328:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572983.043333:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043337:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572983.043345:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043351:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572983.043357:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572983.043366:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572983.043374:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572983.043379:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572983.043385:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572983.043390:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043396:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572983.043401:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572983.043406:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043410:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572983.043418:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788572983.043424:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572983.043430:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572983.043434:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.043439:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572983.043440:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572983.043443:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572983.043448:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788572983.043455:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.043464:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810046592/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572983.043478:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572983.043484:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.043491:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810046592/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572983.043513:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.043520:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13459) 00000020:00000001:0.0:1788572983.489830:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572983.489833:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572983.489835:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572985.537882:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572985.537885:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572985.537888:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572986.881766:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308125855, at 4308125888 00000100:00000010:0.0:1788572986.881786:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788572988.609854:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572988.609860:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572988.609864:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572990.657967:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572990.657971:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572990.657974:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788572990.721719:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c37110399c0] -> 12345-192.168.202.48@tcp (6) 00000800:00000200:0.0:1788572990.721727:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.48@tcp ip 192.168.202.48:988 00000800:00000200:0.0:1788572990.721732:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9c371985bb00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572990.721885:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572990.721900:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572990.721905:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572990.721908:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572992.065747:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308131038, at 4308131072 00000100:00000010:0.0:1788572992.065765:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788572993.729814:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572993.729820:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572993.729824:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572995.777849:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572995.777853:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572995.777855:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572997.185734:0:256930:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308136159, at 4308136192 00000100:00000010:0.0:1788572997.185750:0:256930:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000100:00080000:0.0:1788572998.465784:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572998.465797:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572998.465799:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572998.465803:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572998.465808:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572998.465842:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13465 00000400:00000200:0.0:1788572998.465851:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572998.465856:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572998.465859:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572998.465862:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.465864:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.465866:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.465868:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572998.465871:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572998.465872:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788572998.465875:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572998.465877:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788572998.465879:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788572998.465881:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788572998.465883:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.465884:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.465886:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.465887:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572998.465889:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788572998.465892:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572998.465893:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572998.465896:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572998.465898:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572998.465900:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.465902:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572998.465904:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572998.465909:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788572998.465912:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788572998.465915:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572998.465916:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.465918:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572998.465921:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572998.465928:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572998.465935:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788572998.465940:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.465948:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810048000/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572998.465956:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572998.465960:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.465963:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810048000/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572998.465981:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572998.465983:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572998.465986:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572998.465987:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572998.465988:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572998.465990:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572998.465992:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13465 00000400:00000200:0.0:1788572998.465995:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572998.465998:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572998.465999:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572998.466002:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466004:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466006:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466008:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572998.466010:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572998.466011:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788572998.466013:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572998.466014:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788572998.466015:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788572998.466017:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788572998.466019:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466024:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466036:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466041:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572998.466047:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788572998.466049:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466051:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572998.466055:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572998.466062:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572998.466074:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466084:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572998.466093:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572998.466102:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466111:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572998.466125:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788572998.466134:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572998.466144:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572998.466146:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466150:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572998.466153:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572998.466157:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572998.466162:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788572998.466166:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.466177:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810048128/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572998.466194:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572998.466198:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.466207:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810048128/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572998.466279:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572998.466289:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572998.466297:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572998.466310:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572998.466312:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572998.466315:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572998.466321:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13465 00000400:00000200:0.0:1788572998.466326:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572998.466331:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572998.466333:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572998.466338:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466341:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466344:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466346:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572998.466347:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572998.466348:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788572998.466353:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572998.466360:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788572998.466363:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788572998.466368:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788572998.466376:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466389:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466399:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466404:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572998.466409:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788572998.466415:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466423:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572998.466425:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572998.466433:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572998.466436:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466437:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572998.466439:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572998.466456:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466462:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572998.466464:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788572998.466468:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788572998.466470:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572998.466472:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466473:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572998.466475:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572998.466477:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572998.466480:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788572998.466485:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.466501:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810048256/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572998.466516:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572998.466518:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.466523:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810048256/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572998.466535:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572998.466539:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572998.466550:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572998.466552:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572998.466554:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572998.466560:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572998.466568:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13465 00000400:00000200:0.0:1788572998.466584:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572998.466597:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572998.466602:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572998.466613:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466619:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466626:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466677:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572998.466684:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572998.466694:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788572998.466702:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572998.466709:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788572998.466714:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788572998.466720:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788572998.466737:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466746:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.466750:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466755:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572998.466759:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788572998.466764:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572998.466767:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572998.466772:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572998.466780:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572998.466787:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466791:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572998.466794:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572998.466800:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466802:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572998.466807:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788572998.466812:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788572998.466820:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572998.466826:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.466835:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572998.466841:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572998.466845:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572998.466853:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788572998.466863:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.466875:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810048384/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572998.466892:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572998.466901:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.466920:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810048384/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572998.466951:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572998.466959:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572998.466974:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572998.466981:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572998.466985:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572998.466994:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572998.467003:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13465 00000400:00000200:0.0:1788572998.467023:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788572998.467029:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572998.467037:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572998.467052:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.467061:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.467068:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.467079:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788572998.467086:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572998.467094:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788572998.467098:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572998.467103:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788572998.467113:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788572998.467117:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788572998.467122:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.467124:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572998.467128:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788572998.467134:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788572998.467136:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788572998.467142:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788572998.467148:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572998.467157:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572998.467163:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572998.467173:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.467180:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572998.467183:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572998.467185:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.467187:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572998.467192:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788572998.467202:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788572998.467210:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572998.467215:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.467219:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572998.467225:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572998.467229:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572998.467239:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788572998.467248:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.467259:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810048512/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572998.467288:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572998.467297:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.467311:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810048512/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572998.467338:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.467347:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13475) 00000020:00000001:0.0:1788572998.849834:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572998.849837:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572998.849840:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573000.897883:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573000.897887:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573000.897890:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573002.305737:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308141279, at 4308141312 00000100:00000010:0.0:1788573002.305753:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000100:00080000:0.0:1788573003.585738:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573003.585765:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573003.585766:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573003.585769:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573003.585773:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573003.585779:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13470 00000400:00000200:0.0:1788573003.585787:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573003.585792:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573003.585794:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573003.585797:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585799:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.585801:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585803:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573003.585806:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573003.585807:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573003.585809:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573003.585812:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573003.585814:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573003.585815:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573003.585817:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585818:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.585820:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585821:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573003.585823:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573003.585826:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585827:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573003.585830:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573003.585832:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573003.585834:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.585836:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573003.585837:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573003.585843:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573003.585849:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573003.585852:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573003.585854:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.585856:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573003.585858:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573003.585862:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573003.585869:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573003.585874:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.585881:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810048640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573003.585888:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573003.585892:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.585895:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810048640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573003.585921:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573003.585925:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573003.585928:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573003.585930:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573003.585931:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573003.585934:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573003.585937:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13470 00000400:00000200:0.0:1788573003.585941:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573003.585945:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573003.585947:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573003.585951:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585954:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.585957:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585959:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573003.585962:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573003.585964:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573003.585965:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573003.585975:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573003.585978:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573003.585981:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573003.585987:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585988:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.585992:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.585994:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573003.585998:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573003.586003:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586009:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573003.586012:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573003.586018:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573003.586022:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586025:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573003.586031:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573003.586036:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586040:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573003.586044:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573003.586048:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573003.586051:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573003.586054:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586056:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573003.586060:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573003.586064:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573003.586069:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573003.586074:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.586082:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810048768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573003.586092:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573003.586095:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.586099:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810048768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573003.586128:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573003.586133:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573003.586141:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573003.586144:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573003.586147:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573003.586152:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573003.586157:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13470 00000400:00000200:0.0:1788573003.586165:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573003.586171:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573003.586173:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573003.586177:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586178:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586181:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586184:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573003.586187:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573003.586188:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573003.586192:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573003.586195:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573003.586199:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573003.586201:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573003.586208:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586211:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586216:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586221:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573003.586224:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573003.586230:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586233:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573003.586243:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573003.586248:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573003.586253:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586257:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573003.586260:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573003.586263:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586265:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573003.586269:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573003.586270:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573003.586272:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573003.586273:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586277:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573003.586278:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573003.586280:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573003.586285:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573003.586289:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.586299:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810048896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573003.586313:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573003.586316:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.586321:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810048896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573003.586329:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573003.586331:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573003.586337:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573003.586338:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573003.586350:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573003.586355:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573003.586359:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13470 00000400:00000200:0.0:1788573003.586364:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573003.586369:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573003.586371:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573003.586382:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586388:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586392:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586394:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573003.586396:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573003.586397:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573003.586400:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573003.586402:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573003.586404:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573003.586410:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573003.586416:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586420:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586449:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586455:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573003.586459:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573003.586464:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586467:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573003.586469:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573003.586472:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573003.586475:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586477:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573003.586479:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573003.586480:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586482:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573003.586485:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573003.586488:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573003.586492:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573003.586493:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586498:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573003.586500:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573003.586503:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573003.586515:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573003.586526:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.586534:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810049024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573003.586547:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573003.586551:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.586558:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810049024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573003.586587:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573003.586590:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573003.586598:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573003.586601:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573003.586604:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573003.586607:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573003.586613:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13470 00000400:00000200:0.0:1788573003.586620:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573003.586626:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573003.586628:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573003.586630:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586653:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586655:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586657:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573003.586659:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573003.586660:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573003.586662:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573003.586664:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573003.586665:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573003.586667:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573003.586673:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586678:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573003.586687:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586691:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573003.586695:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573003.586702:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573003.586707:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573003.586711:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573003.586715:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573003.586719:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586722:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573003.586725:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573003.586729:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586731:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573003.586736:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573003.586742:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573003.586745:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573003.586749:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.586753:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573003.586756:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573003.586759:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573003.586764:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573003.586769:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.586775:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810049152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573003.586784:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573003.586787:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.586794:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810049152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573003.586812:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.586816:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13480) 00000020:00000001:0.0:1788573003.969867:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573003.969872:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573003.969874:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573004.546233:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573004.546238:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573004.546240:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573004.546243:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573004.546246:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573004.546250:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13475 00000400:00000200:0.0:1788573004.546257:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573004.546262:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573004.546264:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573004.546267:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546269:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546271:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546273:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573004.546275:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573004.546277:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573004.546278:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573004.546280:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573004.546282:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573004.546283:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573004.546285:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546287:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546289:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546290:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573004.546292:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573004.546294:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546296:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573004.546298:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573004.546300:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573004.546302:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546303:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573004.546310:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573004.546314:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573004.546316:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573004.546318:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573004.546319:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546321:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573004.546322:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573004.546324:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573004.546328:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573004.546331:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.546336:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810049408/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573004.546342:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573004.546344:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.546348:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810049408/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573004.546365:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573004.546367:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573004.546370:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573004.546371:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573004.546372:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573004.546374:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573004.546375:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13475 00000400:00000200:0.0:1788573004.546378:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573004.546380:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573004.546382:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573004.546384:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546386:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546387:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546389:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573004.546391:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573004.546396:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573004.546397:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573004.546399:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573004.546401:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573004.546403:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573004.546405:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546409:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546411:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546416:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573004.546436:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573004.546438:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546440:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573004.546441:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573004.546446:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573004.546449:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546452:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573004.546456:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573004.546461:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546466:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573004.546472:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573004.546477:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573004.546480:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573004.546483:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546485:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573004.546488:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573004.546491:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573004.546497:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573004.546501:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.546509:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810049536/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573004.546517:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573004.546521:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.546534:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810049536/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573004.546544:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573004.546547:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573004.546549:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573004.546553:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573004.546555:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573004.546557:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573004.546559:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13475 00000400:00000200:0.0:1788573004.546564:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573004.546569:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573004.546572:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573004.546580:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546586:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546590:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546597:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573004.546602:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573004.546606:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573004.546610:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573004.546613:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573004.546619:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573004.546623:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573004.546631:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546659:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546661:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546664:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573004.546666:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573004.546672:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546676:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573004.546683:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573004.546688:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573004.546692:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546695:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573004.546698:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573004.546702:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546705:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573004.546709:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573004.546714:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573004.546718:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573004.546719:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546722:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573004.546725:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573004.546727:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573004.546732:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573004.546738:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.546748:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810049664/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573004.546767:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573004.546773:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.546785:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810049664/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573004.546806:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573004.546810:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573004.546814:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573004.546817:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573004.546819:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573004.546822:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573004.546826:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13475 00000400:00000200:0.0:1788573004.546833:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573004.546839:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573004.546842:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573004.546855:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546858:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546863:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546865:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573004.546868:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573004.546871:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573004.546876:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573004.546889:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573004.546895:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573004.546901:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573004.546910:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546917:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.546920:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546922:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573004.546925:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573004.546930:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573004.546935:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573004.546938:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573004.546943:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573004.546946:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546950:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573004.546952:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573004.546956:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546960:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573004.546964:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573004.546970:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573004.546976:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573004.546980:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.546984:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573004.546988:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573004.546992:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573004.546999:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573004.547014:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.547022:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810049792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573004.547034:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573004.547036:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.547044:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810049792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573004.547056:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573004.547061:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573004.547068:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573004.547070:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573004.547073:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573004.547079:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573004.547086:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13475 00000400:00000200:0.0:1788573004.547093:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573004.547099:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573004.547104:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573004.547109:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.547112:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.547114:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.547115:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573004.547117:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573004.547120:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573004.547122:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573004.547126:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573004.547129:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573004.547134:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573004.547141:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.547144:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573004.547154:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573004.547159:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573004.547163:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573004.547170:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573004.547176:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573004.547181:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573004.547187:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573004.547194:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.547199:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573004.547204:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573004.547209:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.547215:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573004.547223:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573004.547226:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573004.547229:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573004.547232:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.547235:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573004.547238:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573004.547241:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573004.547247:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573004.547253:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.547261:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810049920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573004.547276:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573004.547280:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.547287:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810049920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573004.547300:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573004.547306:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13481) 00000020:00000001:0.0:1788573006.017878:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573006.017882:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573006.017885:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573008.385725:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308147359, at 4308147392 00000100:00000010:0.0:1788573008.385738:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573009.089810:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573009.089813:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573009.089816:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573009.729708:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573009.729718:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573009.729721:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573009.729725:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573009.729729:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573009.729746:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13476 00000400:00000200:0.0:1788573009.729755:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573009.729761:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573009.729765:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573009.729767:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729769:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.729771:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729773:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573009.729776:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573009.729777:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573009.729779:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573009.729782:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573009.729784:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573009.729785:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573009.729787:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729789:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.729790:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729792:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573009.729794:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573009.729797:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729814:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573009.729817:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573009.729820:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573009.729822:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.729824:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573009.729826:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573009.729831:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573009.729834:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573009.729837:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573009.729839:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.729841:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573009.729843:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573009.729847:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573009.729853:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573009.729859:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.729866:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810050048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573009.729874:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573009.729878:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.729883:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810050048/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573009.729909:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573009.729914:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573009.729918:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573009.729919:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573009.729921:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573009.729924:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573009.729927:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13476 00000400:00000200:0.0:1788573009.729933:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573009.729937:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573009.729941:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573009.729945:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729957:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.729962:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.729967:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573009.729972:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573009.729975:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573009.729979:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573009.729983:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573009.729988:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573009.729991:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573009.729998:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730004:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730009:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730014:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573009.730017:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573009.730021:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730024:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573009.730027:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573009.730035:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573009.730040:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730047:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573009.730053:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573009.730057:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730061:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573009.730065:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573009.730069:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573009.730071:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573009.730073:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730079:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573009.730083:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573009.730087:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573009.730094:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573009.730101:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.730115:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810050176/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573009.730129:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573009.730131:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.730139:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810050176/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573009.730155:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573009.730158:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573009.730163:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573009.730167:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573009.730171:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573009.730175:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573009.730180:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13476 00000400:00000200:0.0:1788573009.730189:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573009.730193:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573009.730194:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573009.730196:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730198:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730202:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730206:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573009.730211:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573009.730214:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573009.730218:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573009.730222:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573009.730225:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573009.730238:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573009.730251:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730257:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730263:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730275:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573009.730281:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573009.730287:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730291:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573009.730297:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573009.730302:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573009.730307:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730312:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573009.730317:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573009.730322:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730327:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573009.730351:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573009.730356:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573009.730363:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573009.730366:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730371:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573009.730376:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573009.730381:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573009.730388:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573009.730396:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.730408:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810050304/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573009.730418:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573009.730423:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.730431:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810050304/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573009.730447:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573009.730453:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573009.730460:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573009.730464:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573009.730466:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573009.730477:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573009.730484:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13476 00000400:00000200:0.0:1788573009.730492:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573009.730499:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573009.730504:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573009.730511:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730517:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730520:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730525:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573009.730530:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573009.730533:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573009.730538:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573009.730542:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573009.730547:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573009.730552:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573009.730562:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730568:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730574:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730581:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573009.730588:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573009.730596:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730602:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573009.730605:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573009.730610:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573009.730617:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730622:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573009.730628:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573009.730654:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730657:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573009.730661:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573009.730672:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573009.730678:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573009.730682:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730687:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573009.730691:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573009.730695:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573009.730700:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573009.730705:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.730713:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810050432/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573009.730737:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573009.730742:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.730753:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810050432/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573009.730770:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573009.730775:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573009.730779:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573009.730783:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573009.730786:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573009.730791:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573009.730796:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13476 00000400:00000200:0.0:1788573009.730802:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573009.730807:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573009.730811:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573009.730816:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730821:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730823:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730828:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573009.730832:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573009.730834:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573009.730842:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573009.730845:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573009.730849:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573009.730855:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573009.730859:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730862:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573009.730866:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730869:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573009.730870:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573009.730873:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573009.730877:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573009.730880:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573009.730886:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573009.730891:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730895:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573009.730899:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573009.730904:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730907:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573009.730914:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573009.730918:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573009.730922:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573009.730924:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.730930:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573009.730933:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573009.730937:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573009.730944:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573009.730949:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.730958:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810050560/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573009.730971:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573009.730972:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.730977:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810050560/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573009.731000:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573009.731006:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13486) 00000020:00000001:0.0:1788573011.137893:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573011.137896:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573011.137899:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573013.569737:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308152542, at 4308152576 00000100:00000010:0.0:1788573013.569754:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788573014.209931:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573014.209940:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573014.209946:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573014.850096:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573014.850101:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573014.850103:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573014.850112:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850122:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.850125:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.850133:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573014.850138:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573014.850143:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.850151:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810050688:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573014.850161:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573014.850168:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573014.850173:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.850180:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573014.850189:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.850196:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.850207:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573014.850216:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573014.850228:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573014.850254:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810050688, portal 25 00000100:00000001:0.0:1788573014.850257:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573014.850265:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573014.850272:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850279:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573014.850281:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573014.850283:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573014.850286:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810050688, offset 0 00000400:00000200:0.0:1788573014.850291:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573014.850298:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.850303:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573014.850307:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573014.850311:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573014.850314:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573014.850316:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573014.850319:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540526:540526:256:4294967295] 192.168.202.148@tcp LPNI seq info [540491:540491:8:4294967295] 00000400:00000200:0.0:1788573014.850323:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.850330:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573014.850335:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.850340:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573014.850346:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573014.850349:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573014.850360:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.850363:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573014.850406:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573014.850412:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573014.850417:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573014.850418:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573014.850422:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573014.850426:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850437:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.850449:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850463:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.850466:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573014.850479:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.850489:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573014.850493:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308157662 00000100:00000010:0.0:1788573014.850501:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573014.850509:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573014.850518:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308157662 00000100:00000010:0.0:1788573014.850523:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626780. 00000100:00000001:0.0:1788573014.850529:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573014.850533:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573014.850539:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.850545:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573014.850558:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573014/real 1788573014] req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573014.850585:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573014.850589:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573014.850594:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573014.850601:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573014.850604:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573014.850609:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573014.850617:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573014.850664:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850672:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573014.850675:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850680:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.850684:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573014.850692:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850705:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573014.850711:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850724:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573014.850732:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850737:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.850742:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850747:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573014.850752:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573014.850759:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573014.850771:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573014.850789:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573014.850800:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573014.850816:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810050688:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573014.850831:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850837:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.850845:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810050688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850851:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573014.850853:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573014.850862:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.850868:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573014.850871:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573014.850873:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573014.850877:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573014.850881:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573014.850885:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573014.850894:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573014.850895:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573014.850898:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573014.850901:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850904:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.850912:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573014.850914:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.850920:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573014.850922:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573014.850924:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573014.850930:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.850943:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.850946:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.850948:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573014.850951:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573014.850954:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.850961:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810050816:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573014.850966:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573014.850969:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573014.850971:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.850975:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573014.850979:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.850983:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.850993:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573014.850999:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573014.851009:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573014.851015:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810050816, portal 10 00000100:00000001:0.0:1788573014.851020:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573014.851024:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573014.851038:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851058:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573014.851067:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573014.851080:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573014.851087:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810050816, offset 0 00000400:00000200:0.0:1788573014.851097:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851105:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851114:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573014.851122:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573014.851129:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573014.851138:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573014.851145:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851153:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540527:540527:256:4294967295] 192.168.202.148@tcp LPNI seq info [540492:540492:8:4294967295] 00000400:00000200:0.0:1788573014.851166:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851185:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573014.851193:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851200:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573014.851212:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573014.851219:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573014.851264:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851270:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573014.851278:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573014.851291:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573014.851300:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573014.851306:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573014.851314:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573014.851322:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851332:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.851336:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851343:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851344:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573014.851356:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573014.851360:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573014.851364:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573014.851367:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308153881 00000100:00000001:0.0:1788573014.851370:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.851372:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308153881 00000100:00000001:0.0:1788573014.851374:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.851376:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573014.851378:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851381:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573014.851386:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573014/real 1788573014] req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573014.851395:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573014.851396:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573014.851398:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573014.851401:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573014.851403:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573014.851405:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573014.851409:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573014.851418:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851426:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573014.851427:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851428:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.851429:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573014.851432:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851436:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573014.851439:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851452:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573014.851455:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851457:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.851458:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851460:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573014.851462:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573014.851465:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573014.851469:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573014.851471:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573014.851476:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573014.851484:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810050816:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573014.851489:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573014.851493:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851500:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.851501:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.851503:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573014.851506:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573014.851508:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.851511:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810050944:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573014.851515:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573014.851517:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573014.851519:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.851521:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573014.851525:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573014.851527:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.851537:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573014.851540:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573014.851544:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573014.851547:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810050944, portal 10 00000100:00000001:0.0:1788573014.851550:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573014.851551:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573014.851558:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851565:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573014.851567:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573014.851570:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573014.851572:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810050944, offset 0 00000400:00000200:0.0:1788573014.851577:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851581:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851585:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573014.851588:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573014.851591:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573014.851594:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573014.851597:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851599:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540528:540528:256:4294967295] 192.168.202.148@tcp LPNI seq info [540493:540493:8:4294967295] 00000400:00000200:0.0:1788573014.851602:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851607:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573014.851610:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851613:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573014.851615:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573014.851618:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573014.851625:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851627:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573014.851629:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573014.851631:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573014.851634:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573014.851663:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573014.851668:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573014.851673:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851682:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.851686:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851693:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851694:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573014.851699:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573014.851702:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573014.851706:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573014.851709:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308153886 00000100:00000001:0.0:1788573014.851712:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.851714:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308153886 00000100:00000001:0.0:1788573014.851716:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.851719:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573014.851721:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851723:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573014.851727:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573014/real 1788573014] req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573014.851737:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573014.851738:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573014.851740:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573014.851743:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573014.851745:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573014.851747:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573014.851751:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573014.851760:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851771:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573014.851772:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851774:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.851775:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573014.851778:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851782:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573014.851785:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851789:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573014.851791:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851792:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.851792:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851794:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573014.851795:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573014.851796:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573014.851798:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573014.851799:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573014.851802:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573014.851807:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810050944:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573014.851809:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851811:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.851813:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810050816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851817:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573014.851822:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573014.851824:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.851826:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573014.851827:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573014.851828:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573014.851829:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573014.851830:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573014.851832:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573014.851833:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573014.851834:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573014.851836:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573014.851837:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851838:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.851840:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810050944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851845:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573014.851845:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573014.851846:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573014.851848:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573014.851848:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573014.851849:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573014.851850:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573014.851851:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573014.851852:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573014.851853:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573014.851853:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573014.851855:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573014.851856:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851857:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.851860:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573014.851860:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851862:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573014.851863:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573014.851863:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573014.851866:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851870:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.851873:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.851875:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573014.851877:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573014.851878:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.851881:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051072:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573014.851883:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573014.851885:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573014.851886:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.851887:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573014.851889:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573014.851890:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.851893:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573014.851895:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573014.851898:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573014.851900:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051072, portal 4 00000100:00000001:0.0:1788573014.851901:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573014.851902:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573014.851906:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851911:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573014.851912:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573014.851914:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573014.851915:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810051072, offset 0 00000400:00000200:0.0:1788573014.851918:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851920:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851923:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573014.851925:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573014.851927:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573014.851928:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573014.851930:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573014.851932:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540529:540529:256:4294967295] 192.168.202.148@tcp LPNI seq info [540494:540494:8:4294967295] 00000400:00000200:0.0:1788573014.851935:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.851939:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573014.851946:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851948:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573014.851951:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573014.851953:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573014.851961:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.851962:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573014.851964:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573014.851967:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573014.851969:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573014.851970:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573014.851972:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573014.851974:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851979:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.851981:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.851985:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.851986:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573014.851988:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573014.851990:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573014.851992:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573014.851994:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308154018 00000100:00000001:0.0:1788573014.851995:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.851997:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308154018 00000100:00000001:0.0:1788573014.851998:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.852000:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573014.852001:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.852002:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573014.852004:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573014/real 1788573014] req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573014.852013:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573014.852014:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573014.852016:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573014.852017:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573014.852019:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573014.852020:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573014.852022:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573014.852027:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852031:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573014.852032:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852033:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.852034:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573014.852036:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852040:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573014.852043:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852047:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573014.852049:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852050:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.852050:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852052:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573014.852053:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573014.852054:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573014.852056:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573014.852057:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573014.852063:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573014.852068:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051072:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573014.852071:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573014.852073:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852076:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.852077:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.852079:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573014.852080:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573014.852081:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.852083:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051200:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573014.852085:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573014.852086:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573014.852087:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573014.852088:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573014.852089:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.852091:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573014.852093:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573014.852095:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573014.852097:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573014.852099:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051200, portal 4 00000100:00000001:0.0:1788573014.852100:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573014.852101:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573014.852105:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852110:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573014.852111:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573014.852112:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573014.852114:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810051200, offset 0 00000400:00000200:0.0:1788573014.852120:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573014.852122:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.852125:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573014.852127:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573014.852128:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573014.852130:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573014.852131:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573014.852133:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540530:540530:256:4294967295] 192.168.202.148@tcp LPNI seq info [540495:540495:8:4294967295] 00000400:00000200:0.0:1788573014.852136:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573014.852141:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573014.852145:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.852148:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573014.852152:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573014.852155:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573014.852166:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573014.852168:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573014.852171:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573014.852174:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573014.852178:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573014.852180:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573014.852182:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573014.852186:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852195:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.852198:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852204:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.852206:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573014.852210:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573014.852213:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573014.852217:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573014.852225:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308154027 00000100:00000001:0.0:1788573014.852228:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.852230:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308154027 00000100:00000001:0.0:1788573014.852233:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308157662 00000100:00000001:0.0:1788573014.852235:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573014.852249:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.852252:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573014.852257:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573014/real 1788573014] req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573014.852267:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573014.852268:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573014.852270:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573014.852273:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573014.852275:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573014.852278:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573014.852281:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573014.852290:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852297:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573014.852299:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852301:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.852302:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573014.852306:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852312:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573014.852316:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852323:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573014.852333:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852334:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573014.852336:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852338:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573014.852339:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573014.852342:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573014.852346:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573014.852348:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573014.852353:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573014.852361:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051200:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573014.852365:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852367:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.852371:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810051072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852378:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573014.852380:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573014.852382:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573014.852384:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573014.852386:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573014.852387:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573014.852390:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573014.852391:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573014.852394:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573014.852395:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573014.852397:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573014.852400:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573014.852401:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852403:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573014.852406:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810051200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573014.852423:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573014.852424:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573014.852426:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573014.852427:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573014.852428:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573014.852429:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573014.852431:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573014.852432:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573014.852433:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573014.852434:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573014.852435:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573014.852437:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573014.852438:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852439:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.852442:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573014.852442:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573014.852444:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573014.852445:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573016.257885:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573016.257889:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573016.257893:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573018.689720:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308157662, at 4308157696 00000100:00000010:0.0:1788573018.689733:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788573019.329797:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573019.329801:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573019.329805:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573019.970027:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573019.970033:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573019.970038:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573019.970048:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970071:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.970075:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.970082:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573019.970095:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573019.970102:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.970114:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810051328:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573019.970125:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573019.970130:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573019.970134:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.970140:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573019.970162:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573019.970169:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.970182:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573019.970191:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573019.970202:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573019.970213:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051328, portal 25 00000100:00000001:0.0:1788573019.970220:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573019.970224:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351040 : -109714422200576 : ffff9c371ed68700) 00000100:00000040:0.0:1788573019.970238:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970253:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573019.970261:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573019.970268:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573019.970273:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810051328, offset 0 00000400:00000200:0.0:1788573019.970286:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573019.970300:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.970311:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573019.970322:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573019.970331:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573019.970342:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573019.970351:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573019.970363:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540531:540531:256:4294967295] 192.168.202.148@tcp LPNI seq info [540496:540496:8:4294967295] 00000400:00000200:0.0:1788573019.970373:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.970395:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573019.970410:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.970427:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573019.970442:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573019.970450:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573019.970480:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.970485:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573019.970496:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573019.970507:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573019.970517:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573019.970523:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573019.970533:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573019.970552:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970574:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.970584:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970607:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.970611:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573019.970621:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.970633:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573019.970687:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308162783 00000100:00000010:0.0:1788573019.970693:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573019.970701:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573019.970708:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308162783 00000100:00000010:0.0:1788573019.970712:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626900. 00000100:00000001:0.0:1788573019.970715:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573019.970718:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573019.970721:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.970724:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573019.970730:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573019/real 1788573019] req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573019.970755:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573019.970757:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573019.970760:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573019.970762:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573019.970764:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573019.970767:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573019.970772:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573019.970781:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970789:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573019.970791:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970793:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.970795:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573019.970799:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970806:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573019.970811:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970819:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573019.970822:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970824:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.970825:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970828:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573019.970830:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573019.970833:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573019.970838:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573019.970851:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573019.970857:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573019.970872:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810051328:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573019.970879:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970881:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.970886:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68700 x1875440810051328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970893:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573019.970896:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573019.970899:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573019.970903:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573019.970905:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573019.970908:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573019.970910:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573019.970913:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37c00. 02000000:00000001:0.0:1788573019.970916:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573019.970918:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573019.970920:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68700. 00000100:00000001:0.0:1788573019.970923:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573019.970924:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970926:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.970933:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573019.970935:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.970937:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573019.970939:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573019.970940:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573019.970945:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.970953:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.970954:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.970957:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573019.970959:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573019.970962:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.970965:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051456:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573019.970974:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573019.970976:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573019.970978:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.970980:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573019.970982:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573019.970985:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.970989:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573019.970992:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573019.970996:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573019.970999:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051456, portal 10 00000100:00000001:0.0:1788573019.971002:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573019.971003:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287361792 : -109714422189824 : ffff9c371ed6b100) 00000100:00000040:0.0:1788573019.971010:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971017:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573019.971020:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573019.971022:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573019.971025:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810051456, offset 0 00000400:00000200:0.0:1788573019.971030:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971034:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971038:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573019.971042:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573019.971044:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573019.971047:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573019.971050:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971053:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540532:540532:256:4294967295] 192.168.202.148@tcp LPNI seq info [540497:540497:8:4294967295] 00000400:00000200:0.0:1788573019.971058:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971064:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573019.971069:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971072:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573019.971077:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573019.971087:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573019.971099:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971101:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573019.971104:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573019.971108:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573019.971113:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573019.971115:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573019.971118:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573019.971122:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971130:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.971134:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971141:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971159:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573019.971164:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573019.971168:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573019.971172:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573019.971174:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308159001 00000100:00000001:0.0:1788573019.971177:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971180:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308159001 00000100:00000001:0.0:1788573019.971182:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971184:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573019.971186:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971188:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573019.971193:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573019/real 1788573019] req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573019.971202:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573019.971203:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573019.971210:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573019.971213:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573019.971215:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573019.971218:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573019.971223:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573019.971232:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971240:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573019.971242:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971244:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971245:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573019.971249:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971256:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573019.971261:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971269:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573019.971271:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971273:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971275:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971277:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573019.971278:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573019.971281:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573019.971285:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573019.971287:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573019.971291:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573019.971300:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051456:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573019.971310:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573019.971314:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971321:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.971323:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.971325:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573019.971328:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573019.971330:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.971333:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051584:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573019.971337:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573019.971339:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573019.971341:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.971343:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573019.971347:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573019.971350:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.971354:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573019.971357:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573019.971361:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573019.971365:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051584, portal 10 00000100:00000001:0.0:1788573019.971367:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573019.971368:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287359104 : -109714422192512 : ffff9c371ed6a680) 00000100:00000040:0.0:1788573019.971374:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971382:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573019.971384:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573019.971387:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573019.971390:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810051584, offset 0 00000400:00000200:0.0:1788573019.971394:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971398:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971402:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573019.971405:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573019.971413:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573019.971416:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573019.971419:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971422:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540533:540533:256:4294967295] 192.168.202.148@tcp LPNI seq info [540498:540498:8:4294967295] 00000400:00000200:0.0:1788573019.971427:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971434:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573019.971439:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971443:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573019.971447:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573019.971451:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573019.971463:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971464:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573019.971468:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573019.971471:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573019.971475:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573019.971477:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573019.971480:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573019.971484:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971493:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.971497:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971504:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971505:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573019.971509:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573019.971512:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573019.971515:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573019.971517:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308159005 00000100:00000001:0.0:1788573019.971520:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971523:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308159005 00000100:00000001:0.0:1788573019.971531:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971533:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573019.971536:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971538:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573019.971543:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573019/real 1788573019] req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573019.971553:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573019.971554:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573019.971556:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573019.971558:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573019.971560:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573019.971563:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573019.971566:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573019.971575:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971582:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573019.971584:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971586:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971588:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573019.971591:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971599:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573019.971604:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971611:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573019.971615:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971616:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971618:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971626:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573019.971628:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573019.971631:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573019.971653:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573019.971657:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573019.971662:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573019.971671:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051584:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573019.971676:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573019.971680:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971687:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.971689:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.971691:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573019.971694:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573019.971696:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.971700:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051712:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573019.971703:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573019.971705:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573019.971707:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.971709:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573019.971713:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573019.971715:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.971720:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573019.971723:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573019.971727:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573019.971730:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051712, portal 4 00000100:00000001:0.0:1788573019.971732:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573019.971733:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573019.971740:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971753:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573019.971756:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573019.971758:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573019.971762:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810051712, offset 0 00000400:00000200:0.0:1788573019.971766:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971770:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971774:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573019.971777:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573019.971780:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573019.971782:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573019.971785:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573019.971788:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540534:540534:256:4294967295] 192.168.202.148@tcp LPNI seq info [540499:540499:8:4294967295] 00000400:00000200:0.0:1788573019.971793:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.971800:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573019.971805:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971809:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573019.971813:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573019.971816:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573019.971828:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.971830:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573019.971833:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573019.971837:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573019.971840:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573019.971842:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573019.971845:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573019.971849:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971858:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.971861:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971882:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971884:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573019.971888:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573019.971891:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573019.971895:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573019.971897:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308159138 00000100:00000001:0.0:1788573019.971900:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971903:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308159138 00000100:00000001:0.0:1788573019.971905:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.971907:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573019.971909:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.971911:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573019.971916:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573019/real 1788573019] req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573019.971924:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573019.971926:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573019.971928:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573019.971930:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573019.971932:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573019.971935:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573019.971938:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573019.971947:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971953:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573019.971955:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971957:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971958:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573019.971962:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971975:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573019.971980:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.971987:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573019.971990:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971991:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.971993:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.971995:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573019.971997:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573019.972000:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573019.972003:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573019.972006:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573019.972010:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573019.972018:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051712:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573019.972022:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573019.972026:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972033:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.972034:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.972037:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573019.972039:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573019.972041:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.972044:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051840:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573019.972048:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573019.972050:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573019.972056:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573019.972059:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573019.972062:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573019.972064:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573019.972068:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573019.972071:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573019.972075:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573019.972079:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810051840, portal 4 00000100:00000001:0.0:1788573019.972081:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573019.972082:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573019.972089:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972096:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573019.972098:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573019.972100:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573019.972103:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810051840, offset 0 00000400:00000200:0.0:1788573019.972107:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573019.972111:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.972114:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573019.972118:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573019.972120:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573019.972123:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573019.972125:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573019.972128:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540535:540535:256:4294967295] 192.168.202.148@tcp LPNI seq info [540500:540500:8:4294967295] 00000400:00000200:0.0:1788573019.972133:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573019.972139:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573019.972164:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.972168:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573019.972172:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573019.972175:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573019.972187:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573019.972194:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573019.972198:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573019.972202:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573019.972206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573019.972208:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573019.972211:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573019.972215:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972223:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.972227:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972234:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.972235:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573019.972239:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573019.972242:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573019.972245:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573019.972248:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308159147 00000100:00000001:0.0:1788573019.972250:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.972252:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308159147 00000100:00000001:0.0:1788573019.972255:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308162783 00000100:00000001:0.0:1788573019.972257:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573019.972259:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.972261:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573019.972265:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573019/real 1788573019] req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573019.972274:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573019.972275:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573019.972277:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573019.972280:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573019.972282:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573019.972290:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573019.972294:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573019.972303:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972309:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573019.972311:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972313:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.972315:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573019.972318:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972326:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573019.972330:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972337:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573019.972340:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972341:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573019.972343:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972344:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573019.972346:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573019.972349:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573019.972352:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573019.972355:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573019.972359:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573019.972367:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810051840:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573019.972371:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972373:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.972382:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b100 x1875440810051456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972390:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573019.972392:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573019.972394:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573019.972396:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573019.972398:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573019.972399:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573019.972401:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573019.972403:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573019.972406:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573019.972407:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573019.972409:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b100. 00000100:00000001:0.0:1788573019.972412:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573019.972413:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972415:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.972419:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a680 x1875440810051584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972426:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573019.972427:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573019.972429:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573019.972431:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573019.972432:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573019.972433:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573019.972435:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573019.972437:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573019.972439:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573019.972440:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573019.972442:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a680. 00000100:00000001:0.0:1788573019.972444:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573019.972446:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972448:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.972451:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810051712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972458:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573019.972464:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573019.972466:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573019.972469:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573019.972471:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573019.972472:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573019.972474:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573019.972475:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573019.972478:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573019.972479:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573019.972481:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573019.972483:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573019.972484:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972486:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573019.972490:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810051840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573019.972497:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573019.972498:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573019.972500:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573019.972502:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573019.972503:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573019.972505:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573019.972507:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573019.972508:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573019.972511:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573019.972512:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573019.972513:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573019.972516:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573019.972517:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972519:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.972523:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573019.972524:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573019.972527:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573019.972529:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573021.377800:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573021.377804:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573021.377808:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573023.809748:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308162783, at 4308162816 00000100:00000010:0.0:1788573023.809772:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788573024.449831:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573024.449834:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573024.449837:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573025.089741:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.089749:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573025.089752:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.089754:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.089758:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.089763:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13491 00000400:00000200:0.0:1788573025.089771:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573025.089776:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.089778:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.089780:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089783:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.089784:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089786:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573025.089789:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.089791:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573025.089793:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.089796:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573025.089797:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573025.089799:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573025.089801:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089803:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.089804:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089805:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573025.089807:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573025.089809:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089811:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.089830:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.089833:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.089836:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.089837:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573025.089839:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.089844:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573025.089847:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573025.089850:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.089851:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.089854:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.089856:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.089860:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.089866:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573025.089872:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.089879:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810051968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.089887:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.089890:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.089894:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810051968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.089917:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.089920:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.089923:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573025.089924:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.089925:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.089927:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.089929:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13491 00000400:00000200:0.0:1788573025.089932:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573025.089934:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.089936:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.089938:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089939:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.089946:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089949:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573025.089951:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.089953:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573025.089954:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.089956:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573025.089959:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573025.089961:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573025.089964:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089966:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.089967:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089970:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573025.089973:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573025.089976:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573025.089980:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.089982:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.089989:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.089993:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.089998:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.090005:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573025.090012:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090016:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.090022:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573025.090028:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573025.090034:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.090037:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090039:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.090042:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.090045:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.090062:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573025.090066:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.090074:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810052096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.090093:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.090095:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.090103:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810052096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.090118:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.090122:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.090125:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573025.090128:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.090130:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.090133:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.090135:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13491 00000400:00000200:0.0:1788573025.090142:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573025.090148:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.090151:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.090156:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090161:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090165:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090171:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573025.090175:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.090178:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573025.090181:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.090184:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573025.090187:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573025.090194:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573025.090198:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090203:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090207:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090209:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573025.090219:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573025.090225:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090229:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.090232:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.090236:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.090241:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090245:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.090249:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573025.090253:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090258:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.090264:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573025.090267:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573025.090269:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.090272:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090276:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.090279:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.090283:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.090288:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573025.090292:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.090298:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810052224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.090307:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.090311:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.090317:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810052224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.090330:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.090335:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.090338:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573025.090341:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.090344:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.090350:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.090361:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13491 00000400:00000200:0.0:1788573025.090369:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573025.090373:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.090374:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.090377:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090378:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090382:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090386:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573025.090388:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.090391:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573025.090393:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.090396:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573025.090400:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573025.090404:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573025.090411:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090414:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090417:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090421:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573025.090424:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573025.090430:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090436:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.090440:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.090444:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.090448:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090451:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.090453:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573025.090458:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090464:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.090469:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573025.090474:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573025.090476:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.090482:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090484:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.090487:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.090489:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.090493:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573025.090496:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.090503:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810052352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.090513:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.090517:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.090523:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810052352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.090541:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.090545:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.090556:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573025.090560:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.090563:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.090566:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.090569:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13491 00000400:00000200:0.0:1788573025.090575:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573025.090577:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.090579:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.090581:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090583:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090587:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090589:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573025.090591:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.090594:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573025.090596:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.090602:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573025.090607:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573025.090615:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573025.090623:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090628:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.090631:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090659:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573025.090662:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573025.090667:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573025.090671:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.090674:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.090677:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.090683:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090689:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.090691:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573025.090695:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090698:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.090704:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573025.090708:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573025.090711:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.090715:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.090718:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.090720:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.090724:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.090728:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573025.090732:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.090739:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810052480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.090749:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.090753:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.090760:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810052480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.090770:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.090780:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13501) 00000100:00000001:0.0:1788573026.434490:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573026.434495:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573026.434501:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573026.434517:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.434560:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.434570:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.434580:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573026.434585:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573026.434593:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.434607:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810052736:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573026.434623:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573026.434632:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573026.434692:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.434699:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573026.434711:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573026.434720:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.434738:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573026.434752:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573026.434763:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573026.434776:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810052736, portal 25 00000100:00000001:0.0:1788573026.434783:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573026.434787:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573026.434807:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.434863:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573026.434874:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573026.434883:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573026.434892:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810052736, offset 0 00000400:00000200:0.0:1788573026.434911:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573026.434929:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.434943:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573026.434965:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573026.434979:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573026.434989:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573026.434997:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573026.435008:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540541:540541:256:4294967295] 192.168.202.148@tcp LPNI seq info [540506:540506:8:4294967295] 00000400:00000200:0.0:1788573026.435042:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.435068:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573026.435081:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.435091:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573026.435102:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573026.435122:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573026.435156:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.435162:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573026.435170:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573026.435179:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573026.435190:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573026.435197:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573026.435207:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573026.435219:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435253:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.435265:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435294:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.435299:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573026.435308:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.435319:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573026.435326:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308169247 00000100:00000001:0.0:1788573026.435338:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308167902 00000100:00000001:0.0:1788573026.435349:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308169247 00000100:00000001:0.0:1788573026.435366:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308167902 00000100:00000001:0.0:1788573026.435373:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573026.435380:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.435386:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573026.435399:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573026/real 1788573026] req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573026.435420:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573026.435426:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573026.435431:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573026.435439:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573026.435447:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573026.435454:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573026.435461:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573026.435495:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435511:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573026.435513:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435517:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.435518:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573026.435527:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435568:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573026.435583:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435655:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573026.435657:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435658:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.435660:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435662:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573026.435670:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573026.435676:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573026.435689:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573026.435699:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573026.435704:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573026.435711:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810052736:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573026.435716:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435718:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.435723:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810052736/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435728:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573026.435729:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573026.435732:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573026.435735:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573026.435736:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573026.435738:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573026.435739:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573026.435741:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573026.435743:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573026.435744:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573026.435745:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573026.435747:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573026.435747:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435749:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435754:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573026.435755:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.435757:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573026.435758:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573026.435760:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573026.435762:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435775:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.435776:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.435778:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573026.435780:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573026.435781:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.435783:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810052864:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573026.435785:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573026.435787:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573026.435789:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.435790:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573026.435792:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573026.435793:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.435796:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573026.435798:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573026.435801:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573026.435803:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810052864, portal 10 00000100:00000001:0.0:1788573026.435805:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573026.435806:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573026.435810:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435814:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573026.435816:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573026.435818:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573026.435819:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810052864, offset 0 00000400:00000200:0.0:1788573026.435822:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573026.435825:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.435828:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573026.435830:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573026.435831:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573026.435833:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573026.435835:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573026.435840:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540542:540542:256:4294967295] 192.168.202.148@tcp LPNI seq info [540507:540507:8:4294967295] 00000400:00000200:0.0:1788573026.435843:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.435848:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573026.435851:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.435854:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573026.435856:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573026.435858:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573026.435866:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.435867:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573026.435870:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573026.435872:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573026.435874:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573026.435875:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573026.435877:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573026.435880:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435885:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.435887:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435892:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.435893:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573026.435895:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573026.435898:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573026.435900:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573026.435901:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308165466 00000100:00000001:0.0:1788573026.435903:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.435905:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308165466 00000100:00000001:0.0:1788573026.435906:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.435908:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573026.435912:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.435913:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573026.435916:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573026/real 1788573026] req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573026.435922:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573026.435923:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573026.435924:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573026.435926:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573026.435927:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573026.435928:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573026.435930:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573026.435936:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435940:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573026.435941:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435942:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.435943:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573026.435945:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435949:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573026.435951:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435956:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573026.435957:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435958:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.435959:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.435960:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573026.435961:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573026.435962:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573026.435967:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573026.435969:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573026.435972:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573026.435977:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810052864:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573026.435979:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573026.435981:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.435986:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.435987:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.435989:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573026.435990:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573026.435991:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.435993:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810052992:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573026.435996:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573026.435998:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573026.436000:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.436002:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573026.436005:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573026.436007:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.436010:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573026.436013:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573026.436017:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573026.436020:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810052992, portal 10 00000100:00000001:0.0:1788573026.436022:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573026.436037:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573026.436045:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436052:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573026.436061:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573026.436064:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573026.436067:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810052992, offset 0 00000400:00000200:0.0:1788573026.436072:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436076:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436081:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573026.436084:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573026.436087:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573026.436089:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573026.436092:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436095:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540543:540543:256:4294967295] 192.168.202.148@tcp LPNI seq info [540508:540508:8:4294967295] 00000400:00000200:0.0:1788573026.436100:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436107:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573026.436112:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436116:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573026.436120:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573026.436124:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573026.436141:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436144:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573026.436147:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573026.436151:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573026.436155:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573026.436157:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573026.436160:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573026.436164:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436172:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.436175:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436183:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436193:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573026.436197:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573026.436201:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573026.436206:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573026.436208:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308165470 00000100:00000001:0.0:1788573026.436210:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436213:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308165470 00000100:00000001:0.0:1788573026.436215:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436217:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573026.436220:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436222:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573026.436227:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573026/real 1788573026] req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573026.436236:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573026.436238:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573026.436239:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573026.436242:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573026.436244:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573026.436246:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573026.436249:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573026.436259:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436266:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573026.436267:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436270:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.436271:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573026.436274:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436280:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573026.436292:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436300:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573026.436302:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436304:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.436306:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436308:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573026.436309:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573026.436312:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573026.436316:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573026.436318:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573026.436322:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573026.436331:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810052992:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573026.436336:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573026.436340:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436348:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.436349:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.436352:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573026.436354:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573026.436356:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.436359:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810053120:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573026.436363:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573026.436365:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573026.436367:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.436369:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573026.436372:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573026.436375:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.436384:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573026.436387:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573026.436391:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573026.436395:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810053120, portal 4 00000100:00000001:0.0:1788573026.436397:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573026.436398:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573026.436405:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436413:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573026.436415:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573026.436418:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573026.436420:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810053120, offset 0 00000400:00000200:0.0:1788573026.436424:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436428:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436432:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573026.436435:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573026.436438:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573026.436441:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573026.436444:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436447:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540544:540544:256:4294967295] 192.168.202.148@tcp LPNI seq info [540509:540509:8:4294967295] 00000400:00000200:0.0:1788573026.436452:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436458:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573026.436463:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436467:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573026.436471:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573026.436474:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573026.436486:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436488:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573026.436491:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573026.436495:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573026.436498:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573026.436518:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573026.436522:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573026.436526:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436535:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.436538:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436545:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436547:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573026.436551:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573026.436555:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573026.436558:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573026.436561:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308165602 00000100:00000001:0.0:1788573026.436564:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436566:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308165602 00000100:00000001:0.0:1788573026.436568:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436571:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573026.436573:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436575:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573026.436579:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573026/real 1788573026] req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573026.436589:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573026.436590:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573026.436592:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573026.436595:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573026.436596:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573026.436599:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573026.436602:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573026.436611:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436625:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573026.436628:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436630:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.436631:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573026.436657:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436664:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573026.436670:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436678:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573026.436682:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436683:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.436685:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436687:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573026.436689:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573026.436692:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573026.436696:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573026.436698:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573026.436702:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573026.436711:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810053120:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573026.436715:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573026.436719:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436727:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.436729:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.436737:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573026.436739:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573026.436741:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.436744:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810053248:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573026.436748:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573026.436750:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573026.436752:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573026.436754:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573026.436757:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573026.436759:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573026.436763:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573026.436766:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573026.436769:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573026.436772:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810053248, portal 4 00000100:00000001:0.0:1788573026.436775:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573026.436776:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573026.436782:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436790:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573026.436793:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573026.436795:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573026.436798:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810053248, offset 0 00000400:00000200:0.0:1788573026.436802:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436806:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436810:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573026.436813:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573026.436816:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573026.436819:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573026.436821:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573026.436825:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540545:540545:256:4294967295] 192.168.202.148@tcp LPNI seq info [540510:540510:8:4294967295] 00000400:00000200:0.0:1788573026.436829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573026.436836:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573026.436848:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436852:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573026.436856:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573026.436859:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573026.436871:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573026.436873:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0900. 00000400:00000200:0.0:1788573026.436876:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573026.436880:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573026.436883:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31b28 00000400:00000010:0.0:1788573026.436885:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31b28. 00000100:00000001:0.0:1788573026.436888:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573026.436892:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436900:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.436904:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436911:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436912:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573026.436916:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573026.436919:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573026.436923:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573026.436925:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308165612 00000100:00000001:0.0:1788573026.436928:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436931:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308165612 00000100:00000001:0.0:1788573026.436933:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308169247 00000100:00000001:0.0:1788573026.436936:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573026.436938:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.436940:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573026.436945:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573026/real 1788573026] req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573026.436960:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573026.436961:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573026.436963:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8738. 00000400:00000200:0.0:1788573026.436966:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31088 00000400:00000010:0.0:1788573026.436968:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31088. 00000100:00000001:0.0:1788573026.436970:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573026.436975:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573026.436985:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.436992:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573026.436994:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.436996:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.436997:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573026.437001:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437009:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573026.437014:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437022:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573026.437040:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437042:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573026.437043:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437045:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573026.437047:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573026.437050:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573026.437054:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573026.437056:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573026.437068:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573026.437077:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810053248:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573026.437081:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437083:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.437087:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810052864/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437094:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573026.437096:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573026.437098:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573026.437101:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573026.437103:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573026.437104:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573026.437106:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573026.437108:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573026.437111:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573026.437113:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573026.437114:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573026.437117:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573026.437118:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437120:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.437124:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810052992/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437131:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573026.437132:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573026.437134:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573026.437136:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573026.437137:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573026.437138:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573026.437140:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573026.437142:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573026.437144:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573026.437151:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573026.437153:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573026.437155:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573026.437157:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437159:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.437163:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810053120/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437170:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573026.437171:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573026.437173:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573026.437175:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573026.437176:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573026.437178:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573026.437180:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573026.437181:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573026.437184:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573026.437185:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573026.437186:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573026.437188:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573026.437190:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437191:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573026.437194:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810053248/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573026.437201:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573026.437202:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573026.437204:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573026.437206:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573026.437207:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573026.437209:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573026.437211:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573026.437213:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573026.437215:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573026.437217:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573026.437218:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573026.437220:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573026.437222:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437229:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.437233:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573026.437234:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.437236:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573026.437238:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573026.497837:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573026.497841:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573026.497844:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573028.609967:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573028.609974:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573028.609998:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573028.610003:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573028.610006:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573029.569777:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573029.569781:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573029.569785:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573030.273738:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308169247, at 4308169280 00000100:00000010:0.0:1788573030.273762:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000020:00000001:0.0:1788573031.617765:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573031.617768:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573031.617771:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573031.745782:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573031.745794:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573031.745796:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573031.745800:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573031.745805:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573031.745812:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13498 00000400:00000200:0.0:1788573031.745822:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573031.745839:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573031.745842:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573031.745846:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.745850:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.745852:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.745855:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573031.745877:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573031.745879:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573031.745882:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573031.745886:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573031.745889:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573031.745892:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573031.745897:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.745899:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.745902:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.745904:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573031.745908:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573031.745912:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573031.745915:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573031.745918:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573031.745922:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573031.745940:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.745944:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573031.745946:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573031.745954:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573031.745958:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573031.745962:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573031.745965:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.745968:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573031.745971:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573031.745976:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573031.745984:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573031.745991:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.746001:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810053376/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573031.746013:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573031.746017:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.746023:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810053376/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573031.746059:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573031.746063:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573031.746067:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573031.746069:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573031.746072:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573031.746075:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573031.746079:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13498 00000400:00000200:0.0:1788573031.746085:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573031.746090:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573031.746094:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573031.746104:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746111:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.746118:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746125:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573031.746136:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573031.746143:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573031.746151:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573031.746157:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573031.746168:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573031.746175:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573031.746181:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746183:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.746201:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746215:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573031.746222:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573031.746233:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746242:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573031.746256:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573031.746265:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573031.746273:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746278:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573031.746283:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573031.746290:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746293:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573031.746305:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573031.746311:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573031.746325:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573031.746334:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746342:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573031.746347:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573031.746352:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573031.746362:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573031.746370:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.746385:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810053504/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573031.746417:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573031.746429:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.746446:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810053504/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573031.746475:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573031.746484:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573031.746490:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573031.746493:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573031.746497:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573031.746504:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573031.746512:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13498 00000400:00000200:0.0:1788573031.746529:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573031.746540:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573031.746551:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573031.746571:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746577:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.746585:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746592:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573031.746605:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573031.746610:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573031.746614:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573031.746620:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573031.746633:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573031.746711:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573031.746722:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746728:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.746736:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746744:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573031.746752:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573031.746763:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573031.746770:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573031.746774:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573031.746779:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573031.746784:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746792:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573031.746798:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573031.746808:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746826:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573031.746834:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573031.746845:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573031.746855:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573031.746860:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.746872:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573031.746877:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573031.746883:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573031.746894:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573031.746914:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.746947:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810053632/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573031.746981:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573031.746989:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.747003:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810053632/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573031.747017:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573031.747024:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573031.747043:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573031.747050:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573031.747054:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573031.747067:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573031.747078:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13498 00000400:00000200:0.0:1788573031.747096:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573031.747113:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573031.747118:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573031.747130:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747140:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.747151:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747162:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573031.747183:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573031.747193:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573031.747200:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573031.747207:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573031.747217:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573031.747229:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573031.747245:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747251:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.747264:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747272:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573031.747282:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573031.747292:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747296:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573031.747305:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573031.747310:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573031.747317:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747320:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573031.747323:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573031.747327:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747339:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573031.747352:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573031.747362:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573031.747376:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573031.747383:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747392:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573031.747397:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573031.747401:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573031.747411:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573031.747416:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.747428:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810053760/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573031.747462:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573031.747470:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.747481:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810053760/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573031.747514:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573031.747528:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573031.747540:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573031.747546:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573031.747557:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573031.747563:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573031.747576:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13498 00000400:00000200:0.0:1788573031.747603:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573031.747617:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573031.747620:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573031.747630:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747668:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.747679:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747690:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573031.747701:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573031.747708:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573031.747717:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573031.747723:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573031.747726:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573031.747729:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573031.747741:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747743:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573031.747748:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747751:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573031.747757:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573031.747770:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573031.747776:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573031.747781:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573031.747788:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573031.747796:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747804:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573031.747813:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573031.747822:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747833:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573031.747854:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573031.747860:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573031.747866:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573031.747871:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573031.747882:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573031.747886:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573031.747892:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573031.747907:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573031.747917:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.747950:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810053888/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573031.748001:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573031.748013:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.748030:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810053888/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573031.748080:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573031.748091:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13508) 00000020:00000001:0.0:1788573034.689784:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573034.689789:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573034.689791:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573035.585757:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308174559, at 4308174592 00000100:00000010:0.0:1788573035.585772:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788573036.737801:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573036.737805:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573036.737809:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573036.866137:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573036.866144:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573036.866149:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573036.866159:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810054016/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.866173:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.866178:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.866183:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573036.866202:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573036.866208:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.866215:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810054016:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573036.866220:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573036.866226:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573036.866230:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.866233:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573036.866239:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573036.866243:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.866253:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573036.866260:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8738. 00000400:00000010:0.0:1788573036.866271:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573036.866280:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810054016, portal 25 00000100:00000001:0.0:1788573036.866286:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573036.866295:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788573036.866308:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810054016/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.866337:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573036.866343:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573036.866348:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573036.866353:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810054016, offset 0 00000400:00000200:0.0:1788573036.866364:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573036.866380:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.866395:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573036.866408:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573036.866419:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573036.866428:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573036.866434:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573036.866445:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540551:540551:256:4294967295] 192.168.202.148@tcp LPNI seq info [540516:540516:8:4294967295] 00000400:00000200:0.0:1788573036.866456:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.866479:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573036.866494:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573036.866509:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0900. 00000800:00000200:0.0:1788573036.866524:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573036.866564:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.866573:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573036.866579:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308179678 00000100:00000010:0.0:1788573036.866596:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573036.866607:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573036.866621:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308179678 00000100:00000010:0.0:1788573036.866629:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116260c0. 00000100:00000001:0.0:1788573036.866663:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573036.866669:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573036.866677:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.866682:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.866685:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573036.866695:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573036.866698:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573036.866701:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573036.866705:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573036.866708:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573036.866715:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810054144/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.866732:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.866735:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.866737:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573036.866740:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573036.866745:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.866748:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810054144:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573036.866758:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573036.866762:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573036.866764:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.866770:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573036.866776:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573036.866782:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.866797:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573036.866803:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8c08. 00000400:00000010:0.0:1788573036.866812:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573036.866819:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810054144, portal 10 00000100:00000001:0.0:1788573036.866825:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573036.866827:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573036.866851:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810054144/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.866863:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573036.866868:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573036.866871:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573036.866876:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810054144, offset 0 00000400:00000200:0.0:1788573036.866884:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573036.866890:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.866898:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573036.866904:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573036.866909:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573036.866915:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573036.866920:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573036.866925:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540552:540552:255:4294967295] 192.168.202.148@tcp LPNI seq info [540517:540517:7:4294967295] 00000400:00000200:0.0:1788573036.866931:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.866943:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573036.866952:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573036.866959:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0500. 00000800:00000200:0.0:1788573036.866963:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573036.866974:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573036.866981:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573036.866999:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573036.867004:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308175897 00000100:00000001:0.0:1788573036.867012:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867017:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308175897 00000100:00000001:0.0:1788573036.867029:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867036:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573036.867042:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867050:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573036.867060:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810054272/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867077:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.867080:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867086:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573036.867089:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573036.867095:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.867104:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810054272:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573036.867111:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573036.867115:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573036.867118:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867122:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573036.867129:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573036.867135:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.867142:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573036.867147:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8f78. 00000400:00000010:0.0:1788573036.867151:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788573036.867155:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810054272, portal 10 00000100:00000001:0.0:1788573036.867158:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573036.867159:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573036.867168:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810054272/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867175:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573036.867177:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573036.867180:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788573036.867182:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810054272, offset 0 00000400:00000200:0.0:1788573036.867187:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867193:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867205:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573036.867208:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573036.867211:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573036.867213:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573036.867216:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867218:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540553:540553:254:4294967295] 192.168.202.148@tcp LPNI seq info [540518:540518:6:4294967295] 00000400:00000200:0.0:1788573036.867222:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867229:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573036.867234:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573036.867238:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0d00. 00000800:00000200:0.0:1788573036.867241:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573036.867248:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573036.867252:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573036.867255:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573036.867257:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308175901 00000100:00000001:0.0:1788573036.867260:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867262:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308175901 00000100:00000001:0.0:1788573036.867264:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867266:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573036.867268:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867271:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867273:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573036.867276:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573036.867278:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573036.867280:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573036.867282:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573036.867283:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573036.867288:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810054400/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867296:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.867303:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867306:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573036.867308:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573036.867310:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.867313:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810054400:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573036.867317:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573036.867319:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573036.867320:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867322:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573036.867326:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573036.867328:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.867333:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573036.867335:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8210. 00000400:00000010:0.0:1788573036.867340:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573036.867343:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810054400, portal 4 00000100:00000001:0.0:1788573036.867345:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573036.867347:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573036.867354:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810054400/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867361:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573036.867364:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573036.867367:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573036.867369:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810054400, offset 0 00000400:00000200:0.0:1788573036.867375:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867379:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867383:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573036.867386:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573036.867388:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573036.867391:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573036.867393:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867396:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540554:540554:253:4294967295] 192.168.202.148@tcp LPNI seq info [540519:540519:5:4294967295] 00000400:00000200:0.0:1788573036.867400:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867406:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573036.867421:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573036.867425:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0f00. 00000800:00000200:0.0:1788573036.867429:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573036.867436:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573036.867440:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573036.867444:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573036.867446:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308176033 00000100:00000001:0.0:1788573036.867448:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867451:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308176033 00000100:00000001:0.0:1788573036.867453:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867456:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573036.867458:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867461:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573036.867465:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810054528/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867474:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.867476:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867478:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573036.867480:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573036.867483:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.867486:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810054528:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573036.867489:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573036.867492:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573036.867494:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573036.867496:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573036.867500:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573036.867502:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573036.867505:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573036.867508:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d68. 00000400:00000010:0.0:1788573036.867512:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788573036.867521:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810054528, portal 4 00000100:00000001:0.0:1788573036.867523:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573036.867525:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573036.867532:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810054528/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573036.867539:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573036.867541:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573036.867544:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788573036.867547:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810054528, offset 0 00000400:00000200:0.0:1788573036.867551:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867555:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867559:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573036.867562:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573036.867565:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573036.867567:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573036.867570:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573036.867573:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540555:540555:252:4294967295] 192.168.202.148@tcp LPNI seq info [540520:540520:4:4294967295] 00000400:00000200:0.0:1788573036.867578:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573036.867584:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573036.867589:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573036.867594:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0b00. 00000800:00000200:0.0:1788573036.867599:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573036.867605:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573036.867608:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573036.867612:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573036.867614:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308176042 00000100:00000001:0.0:1788573036.867617:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867620:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308176042 00000100:00000001:0.0:1788573036.867622:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308179678 00000100:00000001:0.0:1788573036.867625:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573036.867634:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867663:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867666:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573036.867670:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573036.867672:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573036.867674:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573036.867675:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573036.867677:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573036.867678:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573039.809926:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573039.809931:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573039.809936:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573040.705739:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308179678, at 4308179712 00000100:00000010:0.0:1788573040.705758:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788573041.857873:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573041.857877:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573041.857879:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573041.985754:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573041.985764:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573041.985767:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573041.985770:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573041.985775:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573041.985782:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13508 00000400:00000200:0.0:1788573041.985791:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573041.985797:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573041.985800:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573041.985803:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.985805:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.985807:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.985809:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573041.985812:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573041.985813:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573041.985815:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573041.985832:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573041.985834:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573041.985835:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573041.985838:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.985839:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.985841:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.985842:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573041.985845:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573041.985847:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573041.985849:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573041.985851:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573041.985854:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573041.985856:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.985858:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573041.985860:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573041.985865:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573041.985868:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573041.985870:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573041.985879:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.985882:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573041.985884:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573041.985887:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573041.985894:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573041.985900:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.985907:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810054656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573041.985915:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573041.985920:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.985925:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810054656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573041.985948:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573041.985951:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573041.985966:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573041.985968:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573041.985969:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573041.985972:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573041.985978:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13508 00000400:00000200:0.0:1788573041.985984:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573041.985988:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573041.985991:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573041.985996:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986001:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986005:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986010:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573041.986014:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573041.986016:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573041.986020:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573041.986023:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573041.986025:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573041.986027:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573041.986032:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986033:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986037:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986040:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573041.986046:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573041.986051:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986056:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573041.986057:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573041.986062:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573041.986066:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986071:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573041.986076:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573041.986089:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986093:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573041.986100:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573041.986107:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573041.986113:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573041.986116:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986121:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573041.986124:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573041.986127:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573041.986137:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573041.986144:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.986154:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810054784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573041.986169:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573041.986174:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.986184:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810054784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573041.986201:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573041.986207:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573041.986214:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573041.986218:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573041.986221:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573041.986229:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573041.986234:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13508 00000400:00000200:0.0:1788573041.986244:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573041.986252:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573041.986258:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573041.986264:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986267:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986269:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986279:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573041.986284:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573041.986288:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573041.986292:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573041.986297:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573041.986302:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573041.986307:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573041.986327:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986334:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986341:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986347:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573041.986352:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573041.986360:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986365:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573041.986369:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573041.986375:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573041.986381:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986385:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573041.986389:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573041.986394:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986400:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573041.986407:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573041.986412:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573041.986415:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573041.986416:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986421:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573041.986424:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573041.986428:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573041.986437:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573041.986445:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.986451:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810054912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573041.986475:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573041.986477:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.986482:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810054912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573041.986490:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573041.986493:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573041.986496:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573041.986497:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573041.986499:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573041.986501:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573041.986504:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13508 00000400:00000200:0.0:1788573041.986509:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573041.986513:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573041.986515:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573041.986519:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986521:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986524:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986527:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573041.986531:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573041.986533:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573041.986535:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573041.986541:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573041.986547:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573041.986552:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573041.986561:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986567:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986573:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986577:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573041.986590:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573041.986601:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986608:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573041.986611:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573041.986615:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573041.986619:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986623:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573041.986628:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573041.986634:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986673:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573041.986681:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573041.986684:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573041.986690:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573041.986692:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986695:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573041.986698:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573041.986702:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573041.986708:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573041.986716:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.986726:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810055040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573041.986756:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573041.986761:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.986769:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810055040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573041.986786:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573041.986792:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573041.986796:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573041.986798:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573041.986800:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573041.986805:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573041.986817:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13508 00000400:00000200:0.0:1788573041.986827:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573041.986859:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573041.986865:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573041.986874:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986882:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986889:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986894:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573041.986900:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573041.986903:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573041.986907:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573041.986912:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573041.986917:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573041.986923:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573041.986932:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986937:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573041.986943:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986948:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573041.986954:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573041.986958:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573041.986961:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573041.986963:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573041.986966:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573041.986968:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986970:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573041.986974:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573041.986979:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.986983:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573041.986989:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573041.986996:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573041.987001:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573041.987011:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.987017:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573041.987021:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573041.987026:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573041.987034:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573041.987039:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.987049:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810055168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573041.987064:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573041.987069:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.987077:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810055168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573041.987096:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573041.987102:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13518) 00000020:00000001:0.0:1788573044.929805:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573044.929809:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573044.929812:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573045.825769:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308184798, at 4308184832 00000100:00000010:0.0:1788573045.825785:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573046.977775:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573046.977779:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573046.977781:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573047.105834:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573047.105845:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573047.105848:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573047.105853:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573047.105858:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573047.105867:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13513 00000400:00000200:0.0:1788573047.105877:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573047.105884:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573047.105887:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573047.105891:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.105912:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.105915:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.105917:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573047.105922:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573047.105935:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573047.105941:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573047.105945:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573047.105948:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573047.105950:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573047.105954:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.105956:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.105958:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.105960:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573047.105964:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573047.105967:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573047.105970:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573047.105973:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573047.105977:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573047.105980:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.105983:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573047.105985:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573047.105993:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573047.105997:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573047.106001:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573047.106003:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106006:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573047.106009:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573047.106013:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573047.106021:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573047.106028:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.106038:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810055296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573047.106061:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573047.106067:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.106072:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810055296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573047.106094:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573047.106097:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573047.106101:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573047.106102:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573047.106104:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573047.106107:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573047.106111:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13513 00000400:00000200:0.0:1788573047.106116:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573047.106121:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573047.106126:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573047.106132:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106137:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106140:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106142:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573047.106144:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573047.106146:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573047.106149:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573047.106152:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573047.106158:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573047.106164:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573047.106171:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106176:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106182:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106187:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573047.106200:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573047.106206:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106214:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573047.106219:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573047.106225:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573047.106232:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106237:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573047.106242:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573047.106248:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106253:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573047.106259:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573047.106263:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573047.106268:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573047.106269:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106271:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573047.106272:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573047.106276:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573047.106282:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573047.106288:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.106295:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810055424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573047.106317:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573047.106320:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.106329:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810055424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573047.106343:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573047.106347:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573047.106350:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573047.106351:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573047.106354:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573047.106359:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573047.106371:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13513 00000400:00000200:0.0:1788573047.106378:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573047.106387:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573047.106390:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573047.106397:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106402:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106406:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106408:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573047.106411:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573047.106414:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573047.106417:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573047.106421:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573047.106427:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573047.106432:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573047.106436:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106439:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106442:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106446:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573047.106449:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573047.106451:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106455:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573047.106459:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573047.106463:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573047.106468:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106474:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573047.106477:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573047.106481:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106484:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573047.106488:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573047.106493:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573047.106495:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573047.106502:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106505:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573047.106508:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573047.106510:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573047.106517:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573047.106522:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.106531:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810055552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573047.106550:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573047.106555:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.106564:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810055552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573047.106581:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573047.106583:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573047.106586:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573047.106587:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573047.106589:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573047.106595:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573047.106599:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13513 00000400:00000200:0.0:1788573047.106605:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573047.106608:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573047.106610:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573047.106613:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106616:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106621:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106626:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573047.106631:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573047.106633:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573047.106666:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573047.106668:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573047.106670:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573047.106676:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573047.106680:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106681:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106683:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106688:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573047.106690:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573047.106693:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106697:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573047.106700:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573047.106705:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573047.106709:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106711:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573047.106715:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573047.106722:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106725:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573047.106732:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573047.106736:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573047.106741:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573047.106744:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106747:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573047.106749:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573047.106751:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573047.106755:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573047.106759:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.106765:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810055680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573047.106776:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573047.106779:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.106783:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810055680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573047.106791:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573047.106801:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573047.106806:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573047.106809:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573047.106811:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573047.106816:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573047.106821:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13513 00000400:00000200:0.0:1788573047.106826:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573047.106830:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573047.106834:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573047.106839:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106842:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106846:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106852:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573047.106856:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573047.106858:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573047.106861:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573047.106865:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573047.106868:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573047.106871:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573047.106879:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106884:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573047.106888:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106893:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573047.106897:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573047.106902:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573047.106908:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573047.106913:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573047.106916:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573047.106922:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106926:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573047.106931:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573047.106936:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106939:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573047.106943:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573047.106948:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573047.106951:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573047.106953:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.106955:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573047.106958:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573047.106961:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573047.106964:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573047.106966:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.106973:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810055808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573047.106979:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573047.106983:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.106989:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810055808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573047.107011:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573047.107016:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13523) 00000100:00080000:0.0:1788573048.258342:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573048.258350:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573048.258353:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573048.258356:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573048.258360:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573048.258365:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13518 00000400:00000200:0.0:1788573048.258372:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573048.258379:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573048.258382:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573048.258386:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258389:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.258400:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258403:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573048.258407:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573048.258409:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573048.258411:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573048.258414:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573048.258416:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573048.258419:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573048.258423:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258425:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.258427:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258429:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573048.258432:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573048.258436:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258440:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573048.258442:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573048.258445:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573048.258448:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.258450:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573048.258452:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573048.258457:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573048.258460:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573048.258463:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573048.258465:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.258467:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573048.258469:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573048.258471:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573048.258476:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573048.258480:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.258487:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810056064/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573048.258496:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573048.258503:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.258510:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810056064/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573048.258532:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573048.258535:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573048.258539:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573048.258542:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573048.258543:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573048.258547:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573048.258551:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13518 00000400:00000200:0.0:1788573048.258560:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573048.258564:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573048.258568:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573048.258577:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258582:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.258588:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258594:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573048.258599:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573048.258604:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573048.258611:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573048.258615:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573048.258689:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573048.258695:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573048.258701:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258704:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.258712:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258715:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573048.258721:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573048.258729:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258747:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573048.258752:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573048.258760:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573048.258767:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.258773:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573048.258778:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573048.258784:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.258786:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573048.258793:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573048.258798:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573048.258803:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573048.258806:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.258811:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573048.258814:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573048.258818:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573048.258826:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573048.258833:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.258845:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810056192/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573048.258865:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573048.258870:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.258878:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810056192/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573048.258899:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573048.258904:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573048.258913:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573048.258917:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573048.258921:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573048.258928:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573048.258934:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13518 00000400:00000200:0.0:1788573048.258943:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573048.258958:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573048.258964:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573048.258973:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258979:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.258985:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.258989:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573048.258994:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573048.258997:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573048.259002:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573048.259005:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573048.259008:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573048.259011:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573048.259017:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259021:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.259027:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259032:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573048.259038:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573048.259047:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259053:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573048.259058:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573048.259063:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573048.259069:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259076:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573048.259081:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573048.259087:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259090:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573048.259097:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573048.259103:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573048.259110:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573048.259114:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259119:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573048.259128:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573048.259132:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573048.259139:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573048.259145:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.259162:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810056320/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573048.259178:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573048.259181:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.259187:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810056320/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573048.259204:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573048.259208:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573048.259214:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573048.259216:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573048.259221:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573048.259224:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573048.259228:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13518 00000400:00000200:0.0:1788573048.259235:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573048.259241:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573048.259248:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573048.259256:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259260:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.259264:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259269:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573048.259274:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573048.259276:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573048.259281:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573048.259285:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573048.259290:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573048.259295:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573048.259308:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259315:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.259321:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259327:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573048.259334:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573048.259339:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259345:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573048.259350:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573048.259356:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573048.259360:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259362:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573048.259366:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573048.259372:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259377:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573048.259384:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573048.259389:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573048.259396:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573048.259400:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259405:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573048.259408:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573048.259413:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573048.259420:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573048.259428:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.259439:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810056448/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573048.259457:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573048.259462:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.259473:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810056448/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573048.259499:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573048.259504:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573048.259516:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573048.259520:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573048.259525:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573048.259533:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573048.259543:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13518 00000400:00000200:0.0:1788573048.259557:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573048.259565:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573048.259571:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573048.259578:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259584:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.259588:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259593:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573048.259597:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573048.259600:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573048.259604:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573048.259608:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573048.259614:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573048.259619:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573048.259670:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259675:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573048.259681:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259684:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573048.259689:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573048.259696:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573048.259702:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573048.259707:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573048.259712:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573048.259717:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259722:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573048.259725:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573048.259734:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259738:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573048.259743:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573048.259749:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573048.259756:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573048.259759:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573048.259763:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573048.259767:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573048.259769:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573048.259776:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573048.259783:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.259794:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810056576/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573048.259813:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573048.259818:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.259829:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810056576/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573048.259855:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573048.259862:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13524) 00000020:00000001:0.0:1788573050.049839:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573050.049845:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573050.049849:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573052.097732:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308191071, at 4308191104 00000100:00000010:0.0:1788573052.097746:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000100:00080000:0.0:1788573053.761750:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573053.761759:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573053.761761:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573053.761764:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573053.761768:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573053.761774:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13519 00000400:00000200:0.0:1788573053.761782:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573053.761805:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573053.761808:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573053.761810:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761813:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.761814:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761816:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573053.761819:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573053.761821:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573053.761823:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573053.761826:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573053.761828:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573053.761829:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573053.761831:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761833:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.761834:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761835:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573053.761838:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573053.761840:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761842:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573053.761844:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573053.761846:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573053.761848:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.761850:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573053.761852:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573053.761856:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573053.761858:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573053.761861:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573053.761863:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.761865:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573053.761867:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573053.761870:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573053.761875:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573053.761884:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.761891:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810056704/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573053.761899:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573053.761902:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.761906:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810056704/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573053.761921:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573053.761924:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573053.761926:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573053.761927:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573053.761929:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573053.761930:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573053.761933:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13519 00000400:00000200:0.0:1788573053.761936:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573053.761939:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573053.761941:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573053.761946:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761949:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.761952:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761955:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573053.761958:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573053.761959:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573053.761961:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573053.761965:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573053.761969:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573053.761973:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573053.761980:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761983:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.761991:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.761996:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573053.762001:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573053.762007:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762012:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573053.762015:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573053.762020:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573053.762025:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762027:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573053.762030:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573053.762036:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762039:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573053.762045:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573053.762047:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573053.762049:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573053.762050:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762051:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573053.762053:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573053.762054:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573053.762058:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573053.762062:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.762067:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810056832/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573053.762082:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573053.762087:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.762095:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810056832/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573053.762123:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573053.762129:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573053.762138:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573053.762142:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573053.762154:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573053.762160:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573053.762167:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13519 00000400:00000200:0.0:1788573053.762180:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573053.762191:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573053.762197:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573053.762203:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762209:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762214:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762222:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573053.762226:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573053.762230:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573053.762233:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573053.762237:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573053.762243:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573053.762249:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573053.762258:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762264:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762270:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762277:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573053.762282:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573053.762290:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762297:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573053.762302:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573053.762308:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573053.762314:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762319:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573053.762324:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573053.762328:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762333:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573053.762346:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573053.762353:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573053.762360:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573053.762363:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762367:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573053.762370:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573053.762375:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573053.762382:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573053.762390:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.762401:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810056960/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573053.762434:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573053.762439:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.762449:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810056960/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573053.762472:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573053.762478:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573053.762490:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573053.762493:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573053.762498:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573053.762506:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573053.762512:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13519 00000400:00000200:0.0:1788573053.762538:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573053.762546:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573053.762552:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573053.762560:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762563:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762566:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762568:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573053.762570:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573053.762582:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573053.762586:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573053.762589:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573053.762595:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573053.762601:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573053.762609:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762615:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762620:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762627:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573053.762671:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573053.762680:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762685:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573053.762688:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573053.762692:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573053.762696:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762701:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573053.762705:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573053.762711:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762716:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573053.762723:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573053.762728:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573053.762736:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573053.762740:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762744:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573053.762748:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573053.762752:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573053.762760:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573053.762769:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.762779:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810057088/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573053.762799:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573053.762806:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.762821:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810057088/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573053.762839:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573053.762844:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573053.762851:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573053.762854:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573053.762856:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573053.762858:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573053.762864:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13519 00000400:00000200:0.0:1788573053.762871:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573053.762877:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573053.762883:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573053.762891:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762895:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762899:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762903:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573053.762907:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573053.762910:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573053.762913:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573053.762918:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573053.762924:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573053.762930:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573053.762936:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762941:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573053.762946:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762951:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573053.762956:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573053.762963:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573053.762970:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573053.762981:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573053.762987:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573053.762993:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.762997:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573053.763002:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573053.763006:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.763011:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573053.763017:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573053.763022:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573053.763027:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573053.763030:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573053.763034:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573053.763038:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573053.763041:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573053.763048:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573053.763055:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.763064:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810057216/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573053.763077:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573053.763082:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.763091:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810057216/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573053.763107:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573053.763115:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13530) 00000020:00000001:0.0:1788573055.169833:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573055.169837:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573055.169840:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573057.601746:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308196574, at 4308196608 00000100:00000010:0.0:1788573057.601769:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000100:00000001:0.0:1788573058.882038:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573058.882045:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573058.882049:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573058.882059:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882093:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.882097:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.882103:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573058.882107:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573058.882114:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.882126:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810057344:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573058.882137:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573058.882142:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573058.882148:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.882153:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573058.882160:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573058.882166:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.882179:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573058.882187:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573058.882196:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573058.882207:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057344, portal 25 00000100:00000001:0.0:1788573058.882212:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573058.882214:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573058.882225:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882239:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573058.882246:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573058.882252:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573058.882258:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810057344, offset 0 00000400:00000200:0.0:1788573058.882280:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573058.882294:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.882305:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573058.882315:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573058.882323:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573058.882330:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573058.882335:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573058.882348:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540576:540576:256:4294967295] 192.168.202.148@tcp LPNI seq info [540541:540541:8:4294967295] 00000400:00000200:0.0:1788573058.882353:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.882368:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573058.882383:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.882390:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573058.882405:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573058.882413:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573058.882449:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.882453:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573058.882459:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573058.882468:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573058.882475:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573058.882480:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573058.882485:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573058.882490:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882512:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.882520:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882532:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.882536:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573058.882546:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.882555:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573058.882560:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308201694 00000100:00000010:0.0:1788573058.882568:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573058.882579:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573058.882588:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308201694 00000100:00000010:0.0:1788573058.882595:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626a20. 00000100:00000001:0.0:1788573058.882603:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573058.882609:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573058.882620:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.882626:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573058.882659:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573058/real 1788573058] req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573058.882677:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573058.882680:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573058.882684:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573058.882687:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573058.882688:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573058.882691:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573058.882696:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573058.882709:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882722:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573058.882724:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.882729:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.882731:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573058.882738:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882752:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573058.882758:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882774:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573058.882777:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.882782:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.882785:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.882793:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573058.882799:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573058.882805:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573058.882812:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573058.882835:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573058.882848:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573058.882870:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810057344:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573058.882885:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.882890:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.882907:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810057344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.882933:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573058.882939:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573058.882945:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573058.882954:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573058.882958:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573058.882962:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573058.882970:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573058.882975:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573058.882981:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573058.882984:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573058.882987:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573058.882993:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573058.882997:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883002:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883012:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573058.883017:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883023:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573058.883027:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573058.883031:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573058.883039:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883046:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.883047:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883049:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573058.883051:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573058.883058:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.883061:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057472:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573058.883063:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573058.883065:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573058.883066:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883068:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883070:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573058.883072:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.883075:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573058.883077:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573058.883080:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573058.883082:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057472, portal 10 00000100:00000001:0.0:1788573058.883084:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573058.883085:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573058.883090:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883094:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573058.883096:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573058.883097:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573058.883099:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810057472, offset 0 00000400:00000200:0.0:1788573058.883102:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883106:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883110:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573058.883113:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573058.883116:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573058.883118:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573058.883121:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883124:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540577:540577:256:4294967295] 192.168.202.148@tcp LPNI seq info [540542:540542:8:4294967295] 00000400:00000200:0.0:1788573058.883128:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883136:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573058.883141:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883150:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573058.883155:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573058.883158:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573058.883169:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883171:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573058.883175:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573058.883179:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573058.883183:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573058.883185:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573058.883188:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573058.883192:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883201:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883204:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883209:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883210:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573058.883213:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573058.883216:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573058.883220:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573058.883222:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308197913 00000100:00000001:0.0:1788573058.883225:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883227:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308197913 00000100:00000001:0.0:1788573058.883229:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883231:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573058.883234:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883236:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573058.883240:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573058/real 1788573058] req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573058.883262:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573058.883263:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573058.883265:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573058.883267:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573058.883268:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573058.883270:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573058.883273:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573058.883283:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883289:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573058.883291:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883293:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883295:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573058.883299:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883307:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573058.883311:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883319:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573058.883322:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883323:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883324:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883325:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573058.883327:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573058.883329:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573058.883331:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573058.883333:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573058.883341:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573058.883346:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057472:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573058.883349:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573058.883351:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883355:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.883356:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883358:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573058.883359:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573058.883361:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.883363:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573058.883367:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573058.883369:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573058.883370:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883372:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883375:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573058.883377:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.883381:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573058.883384:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573058.883388:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573058.883391:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057600, portal 10 00000100:00000001:0.0:1788573058.883393:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573058.883394:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573058.883401:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883409:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573058.883411:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573058.883413:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573058.883415:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810057600, offset 0 00000400:00000200:0.0:1788573058.883425:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883448:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883456:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573058.883460:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573058.883463:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573058.883465:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573058.883467:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883469:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540578:540578:256:4294967295] 192.168.202.148@tcp LPNI seq info [540543:540543:8:4294967295] 00000400:00000200:0.0:1788573058.883472:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883477:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573058.883480:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883483:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573058.883486:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573058.883488:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573058.883496:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883498:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573058.883500:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573058.883502:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573058.883505:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573058.883507:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573058.883510:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573058.883514:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883525:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883532:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883534:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573058.883537:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573058.883541:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573058.883545:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573058.883553:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308197917 00000100:00000001:0.0:1788573058.883556:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883558:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308197917 00000100:00000001:0.0:1788573058.883561:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883563:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573058.883566:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883568:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573058.883573:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573058/real 1788573058] req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573058.883583:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573058.883584:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573058.883586:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573058.883589:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573058.883591:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573058.883593:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573058.883597:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573058.883607:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883612:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573058.883614:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883615:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883616:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573058.883619:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883624:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573058.883626:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883630:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573058.883653:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883655:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883656:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883657:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573058.883658:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573058.883660:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573058.883663:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573058.883665:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573058.883667:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573058.883673:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573058.883676:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573058.883678:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883682:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.883683:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883684:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573058.883686:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573058.883688:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.883689:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057728:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573058.883691:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573058.883693:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573058.883694:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.883695:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883697:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573058.883699:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.883701:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573058.883703:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573058.883706:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573058.883708:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057728, portal 4 00000100:00000001:0.0:1788573058.883713:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573058.883714:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573058.883718:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883723:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573058.883724:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573058.883726:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573058.883728:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810057728, offset 0 00000400:00000200:0.0:1788573058.883730:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883733:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883736:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573058.883738:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573058.883740:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573058.883741:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573058.883743:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573058.883745:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540579:540579:256:4294967295] 192.168.202.148@tcp LPNI seq info [540544:540544:8:4294967295] 00000400:00000200:0.0:1788573058.883748:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.883752:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573058.883755:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883758:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573058.883760:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573058.883762:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573058.883770:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.883771:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573058.883773:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573058.883775:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573058.883778:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573058.883780:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573058.883782:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573058.883786:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883806:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883809:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883816:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883818:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573058.883823:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573058.883826:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573058.883830:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573058.883832:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308198050 00000100:00000001:0.0:1788573058.883835:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883836:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308198050 00000100:00000001:0.0:1788573058.883838:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.883839:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573058.883841:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.883842:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573058.883845:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573058/real 1788573058] req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573058.883851:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573058.883852:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573058.883853:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573058.883855:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573058.883856:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573058.883857:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573058.883859:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573058.883865:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883869:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573058.883874:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883876:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883877:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573058.883880:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883884:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573058.883887:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883891:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573058.883893:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883893:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.883894:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883895:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573058.883897:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573058.883898:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573058.883900:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573058.883901:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573058.883904:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573058.883908:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057728:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573058.883911:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883913:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883916:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810057472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883923:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573058.883925:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883927:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573058.883930:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573058.883931:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573058.883937:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573058.883939:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573058.883942:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573058.883944:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573058.883946:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573058.883948:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573058.883951:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573058.883953:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883955:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883959:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810057600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883967:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573058.883968:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883969:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573058.883971:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573058.883972:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573058.883973:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573058.883974:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573058.883975:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573058.883977:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573058.883977:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573058.883978:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573058.883980:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573058.883981:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.883982:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.883984:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810057728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.883989:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573058.883990:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573058.883991:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573058.883994:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573058.883995:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573058.883996:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573058.883998:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573058.884000:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573058.884002:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573058.884008:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573058.884010:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573058.884012:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573058.884014:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884016:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884020:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573058.884021:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.884024:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573058.884025:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573058.884027:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573058.884032:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884039:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.884041:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.884043:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573058.884044:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573058.884046:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.884048:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573058.884050:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573058.884051:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573058.884053:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573058.884054:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573058.884056:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573058.884057:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573058.884060:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573058.884061:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573058.884064:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573058.884066:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057856, portal 4 00000100:00000001:0.0:1788573058.884068:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573058.884069:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788573058.884073:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884077:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573058.884078:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573058.884084:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573058.884085:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810057856, offset 0 00000400:00000200:0.0:1788573058.884088:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573058.884091:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.884093:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573058.884096:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573058.884098:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573058.884099:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573058.884101:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573058.884104:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540580:540580:256:4294967295] 192.168.202.148@tcp LPNI seq info [540545:540545:8:4294967295] 00000400:00000200:0.0:1788573058.884109:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573058.884115:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573058.884120:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.884123:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573058.884128:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573058.884130:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573058.884143:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573058.884145:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573058.884147:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573058.884151:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573058.884154:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573058.884156:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573058.884159:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573058.884163:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884171:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.884175:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884182:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.884184:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573058.884194:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573058.884198:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573058.884202:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573058.884204:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308198059 00000100:00000001:0.0:1788573058.884205:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.884207:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308198059 00000100:00000001:0.0:1788573058.884209:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308201694 00000100:00000001:0.0:1788573058.884210:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573058.884212:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.884213:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573058.884217:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573058/real 1788573058] req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573058.884225:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573058.884227:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573058.884228:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573058.884231:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573058.884232:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573058.884235:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573058.884238:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573058.884247:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884254:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573058.884255:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884257:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.884259:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573058.884263:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884267:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573058.884270:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884283:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573058.884286:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884288:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573058.884289:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884291:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573058.884293:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573058.884295:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573058.884299:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573058.884301:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573058.884306:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573058.884316:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810057856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573058.884320:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884322:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573058.884325:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a300 x1875440810057856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573058.884330:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573058.884331:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573058.884333:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573058.884334:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573058.884336:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573058.884336:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573058.884338:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573058.884339:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573058.884341:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573058.884342:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573058.884343:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a300. 00000100:00000001:0.0:1788573058.884345:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573058.884346:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884351:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.884354:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573058.884355:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573058.884356:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573058.884357:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573060.289789:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573060.289793:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573060.289796:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573062.721753:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308201694, at 4308201728 00000100:00000010:0.0:1788573062.721769:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573064.002016:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573064.002023:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573064.002029:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573064.002041:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002060:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.002065:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.002072:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573064.002078:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573064.002083:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.002093:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810057984:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573064.002103:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573064.002109:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573064.002114:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.002121:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573064.002137:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573064.002146:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.002160:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573064.002169:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573064.002180:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573064.002190:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810057984, portal 25 00000100:00000001:0.0:1788573064.002196:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573064.002200:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287362688 : -109714422188928 : ffff9c371ed6b480) 00000100:00000040:0.0:1788573064.002214:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002251:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573064.002259:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573064.002266:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573064.002273:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810057984, offset 0 00000400:00000200:0.0:1788573064.002286:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573064.002299:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.002311:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573064.002322:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573064.002331:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573064.002350:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573064.002357:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573064.002366:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540581:540581:256:4294967295] 192.168.202.148@tcp LPNI seq info [540546:540546:8:4294967295] 00000400:00000200:0.0:1788573064.002378:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.002395:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573064.002409:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.002420:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573064.002434:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573064.002441:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573064.002472:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.002477:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573064.002486:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573064.002495:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573064.002504:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573064.002509:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573064.002517:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573064.002524:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002541:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.002550:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002579:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.002584:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573064.002593:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.002603:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573064.002609:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308206814 00000100:00000010:0.0:1788573064.002615:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573064.002625:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573064.002677:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308206815 00000100:00000010:0.0:1788573064.002684:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116264e0. 00000100:00000001:0.0:1788573064.002690:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573064.002698:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573064.002704:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.002710:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573064.002721:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573064/real 1788573064] req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573064.002739:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573064.002743:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573064.002748:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573064.002754:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573064.002759:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573064.002761:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573064.002767:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573064.002787:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002801:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573064.002806:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002811:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.002816:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573064.002827:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002853:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573064.002863:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002871:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573064.002875:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002877:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.002879:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002882:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573064.002884:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573064.002888:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573064.002892:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573064.002905:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573064.002911:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573064.002920:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810057984:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573064.002925:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002927:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.002931:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b480 x1875440810057984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.002937:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573064.002939:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573064.002943:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573064.002947:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573064.002949:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573064.002951:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573064.002954:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573064.002956:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36c00. 02000000:00000001:0.0:1788573064.002961:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573064.002969:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573064.002971:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b480. 00000100:00000001:0.0:1788573064.002976:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573064.002978:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002981:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.002989:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573064.002991:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.002994:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573064.002995:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573064.002997:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573064.003003:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003010:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.003012:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003014:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573064.003015:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573064.003017:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.003019:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058112:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573064.003021:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573064.003023:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573064.003024:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003026:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003028:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573064.003029:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.003032:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573064.003035:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573064.003037:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573064.003040:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058112, portal 10 00000100:00000001:0.0:1788573064.003041:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573064.003042:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360000 : -109714422191616 : ffff9c371ed6aa00) 00000100:00000040:0.0:1788573064.003047:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003052:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573064.003054:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573064.003060:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573064.003062:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810058112, offset 0 00000400:00000200:0.0:1788573064.003065:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003068:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003071:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573064.003074:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573064.003076:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573064.003077:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573064.003079:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003081:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540582:540582:256:4294967295] 192.168.202.148@tcp LPNI seq info [540547:540547:8:4294967295] 00000400:00000200:0.0:1788573064.003084:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003089:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573064.003092:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003094:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573064.003097:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573064.003099:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573064.003107:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003108:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573064.003110:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573064.003113:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573064.003115:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573064.003117:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573064.003118:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573064.003121:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003126:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003129:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003133:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003134:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573064.003140:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573064.003143:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573064.003145:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573064.003147:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308203033 00000100:00000001:0.0:1788573064.003149:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308206814 00000100:00000001:0.0:1788573064.003150:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308203033 00000100:00000001:0.0:1788573064.003152:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308206815 00000100:00000001:0.0:1788573064.003153:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573064.003155:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003156:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573064.003158:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573064/real 1788573064] req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573064.003164:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573064.003165:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573064.003166:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573064.003168:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573064.003169:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573064.003170:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573064.003172:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573064.003178:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003182:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573064.003183:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003184:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003185:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573064.003187:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003191:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573064.003193:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003201:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573064.003203:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003204:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003205:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003206:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573064.003207:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573064.003209:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573064.003211:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573064.003212:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573064.003215:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573064.003220:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058112:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573064.003223:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573064.003225:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003229:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.003230:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003231:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573064.003232:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573064.003233:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.003235:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058240:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573064.003237:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573064.003238:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573064.003239:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003240:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003242:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573064.003243:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.003248:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573064.003250:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573064.003252:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573064.003254:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058240, portal 10 00000100:00000001:0.0:1788573064.003256:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573064.003257:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287363584 : -109714422188032 : ffff9c371ed6b800) 00000100:00000040:0.0:1788573064.003261:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003265:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573064.003267:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573064.003268:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573064.003270:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810058240, offset 0 00000400:00000200:0.0:1788573064.003272:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003275:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003277:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573064.003279:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573064.003280:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573064.003282:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573064.003283:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003285:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540583:540583:256:4294967295] 192.168.202.148@tcp LPNI seq info [540548:540548:8:4294967295] 00000400:00000200:0.0:1788573064.003288:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003292:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573064.003295:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003297:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573064.003300:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573064.003301:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573064.003312:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003313:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573064.003315:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573064.003317:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573064.003320:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573064.003323:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573064.003325:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573064.003328:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003351:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003353:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003358:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003359:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573064.003362:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573064.003365:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573064.003367:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573064.003369:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308203037 00000100:00000001:0.0:1788573064.003371:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308206814 00000100:00000001:0.0:1788573064.003372:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308203037 00000100:00000001:0.0:1788573064.003374:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308206815 00000100:00000001:0.0:1788573064.003375:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573064.003376:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003377:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573064.003380:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573064/real 1788573064] req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573064.003385:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573064.003386:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573064.003387:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573064.003389:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573064.003390:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573064.003392:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573064.003394:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573064.003399:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003407:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573064.003408:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003410:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003411:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573064.003413:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003418:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573064.003420:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003424:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573064.003425:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003426:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003427:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003428:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573064.003429:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573064.003431:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573064.003433:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573064.003434:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573064.003436:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573064.003441:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058240:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573064.003443:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573064.003446:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003450:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.003451:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003457:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573064.003458:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573064.003460:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.003462:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058368:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573064.003464:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573064.003465:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573064.003466:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003468:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003470:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573064.003471:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.003474:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573064.003475:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573064.003479:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573064.003482:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058368, portal 4 00000100:00000001:0.0:1788573064.003484:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573064.003485:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788573064.003492:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003498:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573064.003500:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573064.003503:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573064.003506:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810058368, offset 0 00000400:00000200:0.0:1788573064.003510:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003514:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003518:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573064.003521:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573064.003524:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573064.003526:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573064.003529:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003532:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540584:540584:256:4294967295] 192.168.202.148@tcp LPNI seq info [540549:540549:8:4294967295] 00000400:00000200:0.0:1788573064.003537:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003543:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573064.003555:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003559:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573064.003563:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573064.003566:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573064.003575:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003576:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573064.003578:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573064.003580:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573064.003583:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573064.003584:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573064.003586:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573064.003589:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003594:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003596:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003601:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003602:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573064.003604:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573064.003607:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573064.003609:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573064.003611:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308203169 00000100:00000001:0.0:1788573064.003612:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308206814 00000100:00000001:0.0:1788573064.003614:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308203169 00000100:00000001:0.0:1788573064.003616:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308206815 00000100:00000001:0.0:1788573064.003617:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573064.003618:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003619:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573064.003621:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573064/real 1788573064] req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573064.003631:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573064.003632:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573064.003634:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573064.003651:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573064.003652:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573064.003653:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573064.003656:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573064.003662:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003666:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573064.003667:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003668:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003669:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573064.003671:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003675:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573064.003678:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003682:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573064.003684:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003685:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003686:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003687:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573064.003688:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573064.003689:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573064.003691:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573064.003693:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573064.003699:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573064.003704:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058368:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573064.003707:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573064.003709:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003713:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.003714:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003715:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573064.003717:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573064.003718:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.003720:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058496:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573064.003722:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573064.003723:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573064.003724:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573064.003725:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003728:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573064.003729:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573064.003731:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573064.003733:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573064.003735:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573064.003738:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058496, portal 4 00000100:00000001:0.0:1788573064.003739:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573064.003740:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573064.003744:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003749:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573064.003750:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573064.003752:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573064.003753:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810058496, offset 0 00000400:00000200:0.0:1788573064.003759:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003762:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003764:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573064.003766:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573064.003768:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573064.003770:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573064.003771:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573064.003773:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540585:540585:256:4294967295] 192.168.202.148@tcp LPNI seq info [540550:540550:8:4294967295] 00000400:00000200:0.0:1788573064.003776:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573064.003780:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573064.003783:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003785:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573064.003788:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573064.003789:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573064.003797:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573064.003798:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573064.003800:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573064.003802:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573064.003804:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573064.003805:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573064.003807:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573064.003809:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003814:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003816:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003821:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003822:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573064.003823:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573064.003825:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573064.003827:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573064.003832:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308203179 00000100:00000001:0.0:1788573064.003833:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308206814 00000100:00000001:0.0:1788573064.003835:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308203179 00000100:00000001:0.0:1788573064.003836:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308206815 00000100:00000001:0.0:1788573064.003838:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573064.003839:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003841:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573064.003843:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573064/real 1788573064] req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573064.003848:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573064.003849:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573064.003850:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573064.003852:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573064.003853:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573064.003854:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573064.003856:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573064.003862:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003865:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573064.003866:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003867:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003868:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573064.003870:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003874:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573064.003877:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003880:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573064.003885:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003886:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573064.003887:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003888:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573064.003889:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573064.003891:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573064.003893:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573064.003894:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573064.003897:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573064.003901:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058496:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573064.003904:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003905:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003907:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6aa00 x1875440810058112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003911:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573064.003912:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003914:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573064.003915:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573064.003916:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573064.003917:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573064.003918:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573064.003919:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573064.003921:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573064.003922:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573064.003923:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6aa00. 00000100:00000001:0.0:1788573064.003925:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573064.003926:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003927:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003929:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b800 x1875440810058240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003941:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573064.003942:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003944:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573064.003945:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573064.003946:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573064.003947:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573064.003948:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573064.003949:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573064.003950:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573064.003951:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573064.003952:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b800. 00000100:00000001:0.0:1788573064.003953:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573064.003954:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003955:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003957:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a300 x1875440810058368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003960:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573064.003961:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003962:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573064.003963:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573064.003964:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573064.003965:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573064.003966:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573064.003967:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573064.003968:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573064.003968:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573064.003969:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a300. 00000100:00000001:0.0:1788573064.003970:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573064.003971:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003972:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573064.003974:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810058496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573064.003978:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573064.003979:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573064.003983:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573064.003984:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573064.003985:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573064.003986:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573064.003987:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573064.003988:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573064.003989:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573064.003990:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573064.003991:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573064.003992:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573064.003993:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003994:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.003997:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573064.003998:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.003999:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573064.004000:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573065.409943:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573065.409948:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573065.409952:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573066.497905:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573067.841748:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308206814, at 4308206848 00000100:00000010:0.0:1788573067.841765:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573069.122108:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573069.122119:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573069.122125:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573069.122142:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.122168:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.122175:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.122188:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573069.122195:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573069.122203:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.122212:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810058624:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573069.122226:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573069.122232:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573069.122264:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.122268:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573069.122274:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573069.122282:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.122298:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573069.122303:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573069.122311:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573069.122324:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058624, portal 25 00000100:00000001:0.0:1788573069.122327:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573069.122330:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573069.122344:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.122361:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573069.122371:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573069.122379:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573069.122388:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810058624, offset 0 00000400:00000200:0.0:1788573069.122402:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573069.122414:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.122423:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573069.122435:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573069.122443:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573069.122451:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573069.122460:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573069.122470:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540586:540586:256:4294967295] 192.168.202.148@tcp LPNI seq info [540551:540551:8:4294967295] 00000400:00000200:0.0:1788573069.122483:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.122509:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573069.122522:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.122529:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573069.122555:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573069.122564:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573069.122599:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.122616:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573069.122630:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573069.122708:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573069.122717:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573069.122724:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573069.122735:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573069.122743:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.122755:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.122759:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.122767:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.122769:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573069.122774:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.122779:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573069.122782:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308211935 00000100:00000010:0.0:1788573069.122788:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573069.122795:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573069.122806:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308211935 00000100:00000010:0.0:1788573069.122812:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626b40. 00000100:00000001:0.0:1788573069.122818:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573069.122826:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573069.122833:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.122840:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573069.122853:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573069/real 1788573069] req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573069.122878:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573069.122881:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573069.122888:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573069.122894:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573069.122897:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573069.122902:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573069.122916:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573069.122957:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.122979:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573069.122984:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.122994:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123004:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573069.123013:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123033:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573069.123044:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123056:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573069.123060:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123064:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123065:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123072:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573069.123077:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573069.123093:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573069.123102:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573069.123113:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573069.123122:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573069.123167:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810058624:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573069.123175:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123179:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.123182:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810058624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123236:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573069.123253:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573069.123259:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573069.123264:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573069.123272:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573069.123275:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573069.123281:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573069.123285:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573069.123297:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573069.123305:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573069.123315:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573069.123328:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573069.123331:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123346:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123366:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573069.123369:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.123375:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573069.123379:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573069.123381:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573069.123396:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123415:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.123420:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.123432:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573069.123444:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573069.123456:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.123469:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058752:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573069.123472:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573069.123477:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573069.123480:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.123482:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573069.123491:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573069.123503:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.123514:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573069.123526:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573069.123536:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573069.123551:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058752, portal 10 00000100:00000001:0.0:1788573069.123560:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573069.123567:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573069.123588:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123607:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573069.123612:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573069.123614:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573069.123617:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810058752, offset 0 00000400:00000200:0.0:1788573069.123621:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573069.123625:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.123631:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573069.123653:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573069.123655:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573069.123657:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573069.123659:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573069.123661:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540587:540587:256:4294967295] 192.168.202.148@tcp LPNI seq info [540552:540552:8:4294967295] 00000400:00000200:0.0:1788573069.123664:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.123669:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573069.123672:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.123676:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573069.123679:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573069.123681:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573069.123689:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.123690:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573069.123692:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573069.123695:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573069.123697:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573069.123699:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573069.123706:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573069.123709:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123715:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.123717:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123721:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.123722:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573069.123725:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573069.123728:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573069.123730:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573069.123732:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308208154 00000100:00000001:0.0:1788573069.123734:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.123735:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308208154 00000100:00000001:0.0:1788573069.123737:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.123739:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573069.123740:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.123741:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573069.123744:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573069/real 1788573069] req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573069.123749:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573069.123750:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573069.123751:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573069.123752:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573069.123753:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573069.123754:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573069.123756:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573069.123761:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123773:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573069.123774:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123775:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123776:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573069.123779:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123783:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573069.123785:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123790:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573069.123792:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123793:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123793:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123795:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573069.123796:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573069.123797:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573069.123800:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573069.123801:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573069.123803:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573069.123808:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058752:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573069.123810:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573069.123813:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123818:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.123819:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.123823:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573069.123825:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573069.123826:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.123828:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058880:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573069.123831:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573069.123833:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573069.123834:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.123835:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573069.123838:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573069.123839:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.123841:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573069.123843:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573069.123845:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573069.123847:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810058880, portal 10 00000100:00000001:0.0:1788573069.123849:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573069.123849:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573069.123854:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123858:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573069.123859:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573069.123861:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573069.123862:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810058880, offset 0 00000400:00000200:0.0:1788573069.123865:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573069.123867:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.123869:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573069.123871:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573069.123873:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573069.123874:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573069.123876:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573069.123878:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540588:540588:256:4294967295] 192.168.202.148@tcp LPNI seq info [540553:540553:8:4294967295] 00000400:00000200:0.0:1788573069.123881:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.123885:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573069.123891:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.123894:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573069.123896:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573069.123898:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573069.123906:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.123907:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573069.123909:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573069.123911:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573069.123913:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573069.123915:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573069.123916:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573069.123919:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123924:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.123926:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123930:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.123931:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573069.123933:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573069.123936:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573069.123938:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573069.123939:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308208158 00000100:00000001:0.0:1788573069.123941:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.123942:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308208158 00000100:00000001:0.0:1788573069.123944:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.123945:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573069.123946:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.123947:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573069.123949:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573069/real 1788573069] req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573069.123958:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573069.123959:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573069.123961:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573069.123962:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573069.123963:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573069.123965:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573069.123967:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573069.123972:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123975:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573069.123976:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123977:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123978:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573069.123980:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123984:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573069.123987:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.123990:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573069.123992:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123993:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.123994:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.123996:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573069.123997:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573069.123998:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573069.124000:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573069.124001:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573069.124007:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573069.124012:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810058880:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573069.124015:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573069.124017:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124021:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.124022:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.124024:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573069.124025:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573069.124026:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.124028:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059008:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573069.124030:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573069.124031:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573069.124032:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.124033:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124035:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573069.124036:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.124039:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573069.124041:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573069.124043:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573069.124045:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059008, portal 4 00000100:00000001:0.0:1788573069.124047:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573069.124048:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573069.124051:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124056:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573069.124057:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573069.124058:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573069.124060:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810059008, offset 0 00000400:00000200:0.0:1788573069.124066:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573069.124068:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.124071:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573069.124073:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573069.124075:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573069.124077:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573069.124080:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573069.124082:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540589:540589:256:4294967295] 192.168.202.148@tcp LPNI seq info [540554:540554:8:4294967295] 00000400:00000200:0.0:1788573069.124087:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.124093:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573069.124098:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.124101:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573069.124105:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573069.124108:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573069.124120:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.124122:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573069.124125:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573069.124129:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573069.124132:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573069.124134:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573069.124137:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573069.124139:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124145:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124147:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124151:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124153:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573069.124155:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573069.124157:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573069.124160:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573069.124166:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308208290 00000100:00000001:0.0:1788573069.124168:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.124169:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308208290 00000100:00000001:0.0:1788573069.124171:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.124172:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573069.124174:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124175:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573069.124177:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573069/real 1788573069] req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573069.124183:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573069.124184:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573069.124185:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573069.124186:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573069.124188:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573069.124189:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573069.124191:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573069.124196:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124200:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573069.124201:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124202:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.124203:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573069.124205:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124209:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573069.124211:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124216:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573069.124220:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124221:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.124223:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124224:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573069.124225:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573069.124227:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573069.124229:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573069.124230:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573069.124233:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573069.124251:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059008:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573069.124255:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124256:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124258:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810058752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124263:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573069.124264:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124265:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573069.124267:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573069.124268:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573069.124269:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573069.124271:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573069.124272:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573069.124274:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573069.124275:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573069.124276:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573069.124279:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573069.124279:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124281:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124283:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810058880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124296:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573069.124297:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124299:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573069.124300:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573069.124301:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573069.124302:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573069.124303:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573069.124304:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573069.124305:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573069.124306:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573069.124307:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573069.124308:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573069.124309:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124310:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124312:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810059008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124315:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573069.124316:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124317:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573069.124318:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573069.124319:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573069.124319:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573069.124320:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573069.124321:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573069.124323:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573069.124323:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573069.124324:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573069.124325:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573069.124326:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124328:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124331:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573069.124332:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124333:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573069.124335:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573069.124336:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573069.124342:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124347:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.124348:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.124350:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573069.124351:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573069.124352:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.124354:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059136:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573069.124356:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573069.124358:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573069.124359:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573069.124361:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124362:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573069.124363:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573069.124366:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573069.124367:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573069.124370:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573069.124372:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059136, portal 4 00000100:00000001:0.0:1788573069.124374:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573069.124375:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573069.124379:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124383:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573069.124384:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573069.124386:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573069.124387:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810059136, offset 0 00000400:00000200:0.0:1788573069.124390:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573069.124392:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.124394:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573069.124396:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573069.124398:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573069.124400:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573069.124401:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573069.124407:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540590:540590:256:4294967295] 192.168.202.148@tcp LPNI seq info [540555:540555:8:4294967295] 00000400:00000200:0.0:1788573069.124410:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573069.124414:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573069.124417:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.124419:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573069.124422:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573069.124424:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573069.124431:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573069.124433:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573069.124435:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573069.124437:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573069.124439:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573069.124440:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573069.124442:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573069.124444:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124450:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124451:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124456:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124457:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573069.124458:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573069.124461:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573069.124462:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573069.124464:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308208299 00000100:00000001:0.0:1788573069.124465:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.124466:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308208299 00000100:00000001:0.0:1788573069.124468:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308211935 00000100:00000001:0.0:1788573069.124470:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573069.124474:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124475:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573069.124478:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573069/real 1788573069] req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573069.124484:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573069.124485:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573069.124486:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573069.124487:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573069.124488:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573069.124489:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573069.124491:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573069.124496:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124501:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573069.124502:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124503:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.124504:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573069.124506:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124510:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573069.124512:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124516:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573069.124517:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124518:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573069.124519:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124520:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573069.124521:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573069.124523:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573069.124528:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573069.124529:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573069.124532:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573069.124537:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059136:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573069.124540:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124541:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573069.124543:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810059136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573069.124547:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573069.124548:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573069.124549:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573069.124551:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573069.124552:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573069.124553:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573069.124555:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573069.124557:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573069.124559:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573069.124561:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573069.124562:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573069.124565:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573069.124566:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124568:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.124571:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573069.124573:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.124575:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573069.124577:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.466450:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573070.466455:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573070.466458:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573070.466466:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.466482:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.466491:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.466497:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573070.466500:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573070.466505:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.466513:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810059392:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573070.466519:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573070.466524:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573070.466528:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.466531:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573070.466537:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573070.466539:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.466544:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573070.466547:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573070.466557:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573070.466564:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059392, portal 25 00000100:00000001:0.0:1788573070.466567:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573070.466571:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573070.466578:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.466589:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573070.466595:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573070.466598:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573070.466602:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810059392, offset 0 00000400:00000200:0.0:1788573070.466609:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573070.466618:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.466627:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573070.466633:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573070.466695:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573070.466704:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573070.466713:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573070.466726:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540591:540591:256:4294967295] 192.168.202.148@tcp LPNI seq info [540556:540556:8:4294967295] 00000400:00000200:0.0:1788573070.466744:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.466768:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573070.466790:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.466798:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573070.466807:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573070.466813:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573070.466833:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.466835:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573070.466841:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573070.466849:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573070.466857:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573070.466860:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573070.466866:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573070.466873:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.466899:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.466904:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.466917:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.466920:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573070.466926:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.466931:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573070.466935:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308213279 00000100:00000001:0.0:1788573070.466943:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308211935 00000100:00000001:0.0:1788573070.466949:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308213279 00000100:00000001:0.0:1788573070.466953:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308211935 00000100:00000001:0.0:1788573070.466958:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573070.466962:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.466964:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573070.466971:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573070/real 1788573070] req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573070.467005:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573070.467008:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573070.467011:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573070.467015:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573070.467017:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573070.467019:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573070.467022:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573070.467036:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467046:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573070.467049:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467053:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467056:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573070.467062:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467074:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573070.467082:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467093:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573070.467096:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467098:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467101:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467104:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573070.467107:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573070.467110:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573070.467116:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573070.467128:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573070.467135:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573070.467169:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810059392:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573070.467190:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467206:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.467214:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810059392/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467226:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573070.467228:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573070.467232:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573070.467235:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573070.467238:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573070.467242:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573070.467246:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573070.467249:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573070.467254:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573070.467256:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573070.467258:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573070.467271:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573070.467273:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467276:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467283:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573070.467286:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.467292:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573070.467295:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573070.467298:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573070.467304:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467315:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.467318:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467322:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573070.467325:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573070.467330:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.467336:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059520:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573070.467349:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573070.467352:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573070.467355:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467360:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573070.467369:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573070.467374:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.467382:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573070.467386:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573070.467391:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573070.467397:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059520, portal 10 00000100:00000001:0.0:1788573070.467401:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573070.467402:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573070.467407:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467416:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573070.467421:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573070.467425:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573070.467429:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810059520, offset 0 00000400:00000200:0.0:1788573070.467436:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573070.467442:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.467448:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573070.467453:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573070.467456:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573070.467460:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573070.467463:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573070.467469:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540592:540592:256:4294967295] 192.168.202.148@tcp LPNI seq info [540557:540557:8:4294967295] 00000400:00000200:0.0:1788573070.467474:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.467484:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573070.467496:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.467501:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573070.467506:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573070.467516:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573070.467533:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.467534:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573070.467536:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573070.467540:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573070.467543:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573070.467544:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573070.467547:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573070.467550:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467555:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.467557:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467562:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.467563:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573070.467566:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573070.467570:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573070.467574:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573070.467576:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308209497 00000100:00000001:0.0:1788573070.467580:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.467582:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308209497 00000100:00000001:0.0:1788573070.467584:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.467586:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573070.467589:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.467590:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573070.467595:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573070/real 1788573070] req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573070.467604:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573070.467605:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573070.467613:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573070.467616:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573070.467617:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573070.467620:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573070.467624:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573070.467634:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467655:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573070.467657:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467658:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467660:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573070.467663:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467668:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573070.467671:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467675:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573070.467677:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467678:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467678:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467680:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573070.467681:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573070.467682:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573070.467684:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573070.467686:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573070.467688:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573070.467693:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059520:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573070.467700:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573070.467703:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467708:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.467709:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467710:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573070.467712:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573070.467713:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.467715:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059648:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573070.467717:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573070.467718:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573070.467719:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467721:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573070.467723:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573070.467725:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.467727:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573070.467729:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573070.467731:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573070.467734:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059648, portal 10 00000100:00000001:0.0:1788573070.467735:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573070.467736:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573070.467740:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467744:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573070.467746:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573070.467747:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573070.467749:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810059648, offset 0 00000400:00000200:0.0:1788573070.467751:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573070.467754:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.467756:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573070.467758:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573070.467763:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573070.467765:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573070.467767:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573070.467769:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540593:540593:256:4294967295] 192.168.202.148@tcp LPNI seq info [540558:540558:8:4294967295] 00000400:00000200:0.0:1788573070.467772:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.467777:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573070.467780:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.467782:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573070.467785:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573070.467786:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573070.467794:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.467795:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573070.467797:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573070.467799:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573070.467802:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573070.467803:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573070.467805:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573070.467807:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467812:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.467814:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467819:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.467820:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573070.467822:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573070.467824:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573070.467826:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573070.467827:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308209502 00000100:00000001:0.0:1788573070.467829:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.467830:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308209502 00000100:00000001:0.0:1788573070.467836:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.467838:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573070.467839:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.467840:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573070.467843:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573070/real 1788573070] req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573070.467852:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573070.467853:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573070.467855:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573070.467857:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573070.467859:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573070.467861:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573070.467865:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573070.467874:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467881:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573070.467882:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467884:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467886:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573070.467890:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467898:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573070.467902:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467909:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573070.467912:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467914:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.467915:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.467921:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573070.467923:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573070.467925:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573070.467927:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573070.467929:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573070.467932:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573070.467937:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059648:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573070.467940:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573070.467942:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467946:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.467947:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467949:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573070.467950:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573070.467952:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.467954:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059776:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573070.467955:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573070.467957:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573070.467957:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.467959:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573070.467961:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573070.467962:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.467965:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573070.467967:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573070.467970:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573070.467972:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059776, portal 4 00000100:00000001:0.0:1788573070.467973:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573070.467974:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573070.467978:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.467986:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573070.467988:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573070.467989:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573070.467991:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810059776, offset 0 00000400:00000200:0.0:1788573070.467993:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573070.467996:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.467999:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573070.468001:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573070.468002:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573070.468004:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573070.468006:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573070.468008:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540594:540594:256:4294967295] 192.168.202.148@tcp LPNI seq info [540559:540559:8:4294967295] 00000400:00000200:0.0:1788573070.468011:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.468015:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573070.468018:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.468020:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573070.468023:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573070.468025:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573070.468032:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.468033:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573070.468035:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573070.468038:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573070.468040:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573070.468041:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573070.468043:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573070.468045:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468050:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468052:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468074:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468075:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573070.468077:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573070.468080:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573070.468082:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573070.468084:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308209634 00000100:00000001:0.0:1788573070.468085:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.468087:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308209634 00000100:00000001:0.0:1788573070.468088:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.468089:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573070.468091:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468092:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573070.468094:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573070/real 1788573070] req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573070.468099:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573070.468100:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573070.468102:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573070.468103:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573070.468104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573070.468106:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573070.468108:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573070.468113:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468117:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573070.468118:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468119:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.468120:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573070.468122:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468129:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573070.468132:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468136:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573070.468138:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468139:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.468139:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468141:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573070.468142:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573070.468144:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573070.468146:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573070.468148:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573070.468150:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573070.468154:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059776:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573070.468157:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468158:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468160:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810059520/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468164:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573070.468165:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573070.468166:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573070.468167:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573070.468168:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573070.468169:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573070.468170:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573070.468171:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573070.468176:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573070.468177:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573070.468178:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573070.468180:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573070.468181:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468182:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468184:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810059648/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468189:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573070.468189:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573070.468190:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573070.468205:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573070.468206:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573070.468207:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573070.468208:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573070.468209:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573070.468211:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573070.468212:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573070.468212:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573070.468214:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573070.468215:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468216:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468218:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810059776/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468222:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573070.468223:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573070.468224:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573070.468225:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573070.468226:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573070.468226:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=3 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573070.468228:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573070.468229:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573070.468230:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573070.468231:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573070.468232:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573070.468234:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573070.468239:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468240:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468243:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573070.468244:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468245:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573070.468247:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573070.468248:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573070.468250:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468255:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.468256:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.468257:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573070.468259:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573070.468260:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.468262:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059904:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573070.468263:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573070.468265:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573070.468266:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573070.468267:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573070.468269:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573070.468270:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573070.468273:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573070.468274:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573070.468277:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573070.468279:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810059904, portal 4 00000100:00000001:0.0:1788573070.468280:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573070.468281:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573070.468285:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468289:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573070.468291:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573070.468292:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573070.468294:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810059904, offset 0 00000400:00000200:0.0:1788573070.468296:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573070.468302:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.468305:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573070.468306:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573070.468308:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573070.468310:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573070.468312:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573070.468314:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540595:540595:256:4294967295] 192.168.202.148@tcp LPNI seq info [540560:540560:8:4294967295] 00000400:00000200:0.0:1788573070.468317:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573070.468321:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573070.468323:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.468325:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573070.468328:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573070.468330:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573070.468343:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573070.468345:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573070.468348:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573070.468352:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573070.468356:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573070.468358:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573070.468361:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573070.468365:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468373:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468376:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468383:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468385:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573070.468388:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573070.468392:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573070.468395:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573070.468403:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308209643 00000100:00000001:0.0:1788573070.468406:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.468409:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308209643 00000100:00000001:0.0:1788573070.468411:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308213279 00000100:00000001:0.0:1788573070.468413:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573070.468416:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468418:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573070.468423:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573070/real 1788573070] req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573070.468432:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573070.468434:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573070.468435:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573070.468438:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573070.468440:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573070.468443:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573070.468446:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573070.468454:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468458:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573070.468459:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468460:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.468461:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573070.468464:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468468:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573070.468471:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468475:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573070.468483:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468485:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573070.468486:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468488:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573070.468490:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573070.468492:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573070.468496:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573070.468498:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573070.468502:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573070.468511:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810059904:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573070.468515:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468517:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573070.468521:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810059904/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573070.468527:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573070.468529:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573070.468531:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573070.468534:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573070.468535:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573070.468537:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=2 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573070.468539:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573070.468541:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573070.468544:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573070.468545:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573070.468546:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573070.468550:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573070.468551:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468553:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.468556:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573070.468557:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573070.468558:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573070.468563:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573070.529850:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573070.529855:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573070.529859:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573074.305705:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308213279, at 4308213312 00000100:00000010:0.0:1788573074.305712:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788573074.625824:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573074.625827:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573074.625830:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573075.657271:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.657276:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.657278:0:2404:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.657288:0:2404:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4000 x1875440810060160/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 712/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.657327:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.657331:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788573075.657338:0:2404:0:(client.c:1853:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9c371f1c4000 x1875440810060160/t0(0) o103->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 712/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.657349:0:2404:0:(client.c:1858:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657353:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657358:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.657362:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657367:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657372:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657383:0:2404:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573075.657388:0:2404:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657395:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.657399:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.657402:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657409:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.657414:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657419:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657426:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657456:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.657461:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.657476:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657484:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.657488:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.657494:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.657498:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.778100:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.778108:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.778114:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.778127:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778142:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.778144:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.778151:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573075.778155:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573075.778157:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.778165:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810060288:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573075.778172:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573075.778177:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573075.778181:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.778184:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573075.778192:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.778195:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.778203:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573075.778208:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573075.778219:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573075.778228:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810060288, portal 25 00000100:00000001:0.0:1788573075.778231:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573075.778233:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287359104 : -109714422192512 : ffff9c371ed6a680) 00000100:00000040:0.0:1788573075.778241:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778251:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573075.778255:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573075.778259:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573075.778261:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810060288, offset 0 00000400:00000200:0.0:1788573075.778281:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573075.778294:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.778307:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573075.778318:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573075.778327:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573075.778335:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573075.778342:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573075.778350:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540596:540596:256:4294967295] 192.168.202.148@tcp LPNI seq info [540561:540561:8:4294967295] 00000400:00000200:0.0:1788573075.778359:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.778374:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573075.778385:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.778394:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573075.778406:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573075.778412:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573075.778433:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.778437:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573075.778445:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573075.778454:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573075.778460:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573075.778463:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573075.778467:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573075.778482:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778499:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.778506:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778520:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.778523:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573075.778530:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.778537:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573075.778541:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308218590 00000100:00000010:0.0:1788573075.778554:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788573075.778567:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573075.778578:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308218590 00000100:00000010:0.0:1788573075.778584:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626420. 00000100:00000001:0.0:1788573075.778588:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573075.778594:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573075.778600:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.778602:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573075.778609:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573075/real 1788573075] req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573075.778667:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573075.778670:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573075.778673:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573075.778677:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573075.778680:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573075.778685:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573075.778691:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573075.778704:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778719:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573075.778722:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778725:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.778728:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573075.778740:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778752:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573075.778761:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778771:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573075.778783:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778786:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.778789:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778792:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573075.778796:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573075.778800:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573075.778807:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573075.778819:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573075.778827:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573075.778841:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810060288:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573075.778849:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778853:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.778859:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a680 x1875440810060288/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778871:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573075.778876:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573075.778879:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.778884:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573075.778888:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573075.778891:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573075.778895:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573075.778899:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37c00. 02000000:00000001:0.0:1788573075.778908:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573075.778909:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573075.778911:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a680. 00000100:00000001:0.0:1788573075.778914:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573075.778916:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778920:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.778929:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573075.778933:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.778936:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.778947:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.778951:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.778959:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.778974:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.778976:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.778981:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573075.778986:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573075.779003:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.779011:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060416:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573075.779019:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573075.779022:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573075.779025:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.779027:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573075.779032:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.779037:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.779042:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573075.779045:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573075.779050:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573075.779056:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810060416, portal 10 00000100:00000001:0.0:1788573075.779060:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573075.779063:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287361792 : -109714422189824 : ffff9c371ed6b100) 00000100:00000040:0.0:1788573075.779071:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779089:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573075.779095:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573075.779100:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573075.779107:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810060416, offset 0 00000400:00000200:0.0:1788573075.779118:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779128:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779135:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573075.779141:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573075.779144:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573075.779154:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573075.779159:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779164:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540597:540597:256:4294967295] 192.168.202.148@tcp LPNI seq info [540562:540562:8:4294967295] 00000400:00000200:0.0:1788573075.779171:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779184:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573075.779194:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779198:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573075.779206:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573075.779210:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573075.779234:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779236:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573075.779240:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573075.779245:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573075.779249:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573075.779252:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573075.779256:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573075.779262:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779275:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.779279:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779292:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779297:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573075.779303:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573075.779312:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573075.779321:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573075.779326:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308214809 00000100:00000001:0.0:1788573075.779330:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779337:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308214809 00000100:00000001:0.0:1788573075.779346:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779349:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573075.779355:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779358:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573075.779366:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573075/real 1788573075] req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573075.779380:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573075.779384:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573075.779386:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573075.779391:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573075.779393:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573075.779396:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573075.779402:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573075.779412:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779418:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573075.779421:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779423:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.779425:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573075.779427:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779432:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573075.779435:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779439:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573075.779441:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779442:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.779443:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779448:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573075.779450:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573075.779452:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573075.779455:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573075.779457:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573075.779460:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573075.779465:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060416:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573075.779468:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.779470:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779474:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.779475:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.779476:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573075.779478:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573075.779479:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.779481:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060544:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573075.779484:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573075.779485:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573075.779486:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.779488:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573075.779490:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573075.779492:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.779494:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573075.779496:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573075.779499:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573075.779501:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810060544, portal 10 00000100:00000001:0.0:1788573075.779503:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573075.779504:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351040 : -109714422200576 : ffff9c371ed68700) 00000100:00000040:0.0:1788573075.779508:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779518:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573075.779520:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573075.779521:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573075.779523:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810060544, offset 0 00000400:00000200:0.0:1788573075.779526:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779528:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779531:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573075.779534:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573075.779536:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573075.779537:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573075.779539:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779541:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540598:540598:256:4294967295] 192.168.202.148@tcp LPNI seq info [540563:540563:8:4294967295] 00000400:00000200:0.0:1788573075.779544:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779549:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573075.779552:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779555:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573075.779558:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573075.779560:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573075.779568:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779569:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573075.779571:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573075.779574:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573075.779576:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573075.779577:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573075.779579:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573075.779581:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779587:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.779589:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779602:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779603:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573075.779606:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573075.779608:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573075.779611:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573075.779613:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308214813 00000100:00000001:0.0:1788573075.779614:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779616:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308214813 00000100:00000001:0.0:1788573075.779618:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779619:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573075.779620:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779621:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573075.779624:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573075/real 1788573075] req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573075.779629:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573075.779630:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573075.779631:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573075.779633:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573075.779634:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573075.779653:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573075.779655:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573075.779661:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779665:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573075.779666:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779667:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.779668:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573075.779670:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779679:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573075.779682:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779687:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573075.779689:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779690:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.779691:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779693:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573075.779694:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573075.779695:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573075.779697:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573075.779698:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573075.779701:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573075.779705:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060544:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573075.779708:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779709:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.779711:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b100 x1875440810060416/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779723:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573075.779724:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573075.779728:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.779730:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573075.779732:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573075.779733:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573075.779735:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573075.779736:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573075.779741:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573075.779742:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573075.779744:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b100. 00000100:00000001:0.0:1788573075.779746:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573075.779747:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779748:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.779751:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68700 x1875440810060544/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779755:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573075.779756:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573075.779758:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573075.779760:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573075.779761:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573075.779761:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573075.779762:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573075.779763:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573075.779765:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573075.779765:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573075.779766:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68700. 00000100:00000001:0.0:1788573075.779767:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573075.779768:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779769:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779772:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573075.779773:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779774:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.779775:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.779776:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.779778:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779782:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.779783:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.779784:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573075.779785:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573075.779787:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.779788:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060672:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573075.779795:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573075.779797:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573075.779798:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.779800:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573075.779801:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573075.779803:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.779806:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573075.779808:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573075.779811:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573075.779813:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810060672, portal 4 00000100:00000001:0.0:1788573075.779815:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573075.779816:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287359104 : -109714422192512 : ffff9c371ed6a680) 00000100:00000040:0.0:1788573075.779821:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779825:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573075.779827:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573075.779828:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573075.779830:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810060672, offset 0 00000400:00000200:0.0:1788573075.779833:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779835:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779838:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573075.779840:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573075.779841:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573075.779843:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573075.779845:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573075.779847:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540599:540599:256:4294967295] 192.168.202.148@tcp LPNI seq info [540564:540564:8:4294967295] 00000400:00000200:0.0:1788573075.779850:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.779854:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573075.779857:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779859:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573075.779862:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573075.779867:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573075.779875:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.779876:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573075.779878:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573075.779881:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573075.779884:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573075.779885:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573075.779887:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573075.779889:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779895:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.779897:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779901:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779902:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573075.779904:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573075.779907:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573075.779909:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573075.779911:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308214946 00000100:00000001:0.0:1788573075.779913:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779914:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308214946 00000100:00000001:0.0:1788573075.779916:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.779917:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573075.779919:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.779920:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573075.779922:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573075/real 1788573075] req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573075.779926:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573075.779927:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573075.779932:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573075.779933:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573075.779935:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573075.779936:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573075.779939:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573075.779947:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779954:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573075.779956:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.779957:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.779959:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573075.779963:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779971:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573075.779976:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.779983:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573075.779986:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780003:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.780005:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780007:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573075.780009:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573075.780012:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573075.780014:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573075.780016:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573075.780019:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573075.780025:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060672:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573075.780034:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573075.780037:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780042:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.780043:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.780045:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573075.780046:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573075.780047:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.780049:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060800:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573075.780051:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573075.780052:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573075.780054:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573075.780055:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573075.780057:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.780059:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573075.780061:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573075.780063:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573075.780065:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573075.780068:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810060800, portal 4 00000100:00000001:0.0:1788573075.780069:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573075.780070:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573075.780075:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780081:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573075.780083:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573075.780086:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573075.780088:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810060800, offset 0 00000400:00000200:0.0:1788573075.780092:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573075.780097:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.780101:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573075.780104:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573075.780112:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573075.780114:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573075.780118:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573075.780121:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540600:540600:256:4294967295] 192.168.202.148@tcp LPNI seq info [540565:540565:8:4294967295] 00000400:00000200:0.0:1788573075.780126:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573075.780133:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573075.780136:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.780139:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573075.780142:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573075.780144:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573075.780152:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573075.780153:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573075.780155:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573075.780157:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573075.780160:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573075.780161:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573075.780163:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573075.780166:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780172:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.780174:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780179:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.780180:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573075.780182:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573075.780184:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573075.780186:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573075.780188:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308214955 00000100:00000001:0.0:1788573075.780190:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.780191:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308214955 00000100:00000001:0.0:1788573075.780197:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308218590 00000100:00000001:0.0:1788573075.780198:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573075.780200:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.780201:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573075.780204:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573075/real 1788573075] req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573075.780210:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573075.780211:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573075.780213:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573075.780214:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573075.780216:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573075.780217:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573075.780219:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573075.780224:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780228:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573075.780229:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780230:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.780231:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573075.780233:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780238:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573075.780240:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780244:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573075.780246:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780247:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573075.780248:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780252:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573075.780253:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573075.780255:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573075.780258:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573075.780259:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573075.780262:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573075.780268:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810060800:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573075.780270:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780271:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.780273:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a680 x1875440810060672/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780277:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573075.780278:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573075.780279:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573075.780280:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573075.780281:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573075.780282:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=4 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573075.780283:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573075.780284:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573075.780286:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573075.780287:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573075.780288:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a680. 00000100:00000001:0.0:1788573075.780290:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573075.780291:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780293:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573075.780296:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810060800/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573075.780303:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573075.780305:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573075.780306:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573075.780315:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573075.780317:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573075.780318:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573075.780320:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573075.780322:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573075.780325:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573075.780326:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573075.780327:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573075.780330:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573075.780331:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780333:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.780337:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573075.780339:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.780341:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573075.780343:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573078.721837:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573078.721841:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573078.721843:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573079.617716:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308218590, at 4308218624 00000100:00000010:0.0:1788573079.617734:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626180. 00000020:00000001:0.0:1788573082.817814:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573082.817818:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573082.817820:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573084.737791:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308223710, at 4308223744 00000100:00000010:0.0:1788573084.737819:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 00000100:00080000:0.0:1788573086.017761:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.017773:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573086.017776:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.017780:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.017785:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.017792:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13552 00000400:00000200:0.0:1788573086.017801:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573086.017807:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.017811:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.017829:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017831:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.017833:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017834:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573086.017837:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.017839:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573086.017841:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.017844:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573086.017845:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573086.017847:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573086.017849:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017851:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.017852:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017854:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573086.017856:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573086.017858:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017860:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.017862:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.017865:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.017867:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.017868:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573086.017870:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.017875:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573086.017877:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573086.017880:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.017881:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.017883:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.017886:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.017890:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.017896:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573086.017902:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.017909:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810061696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.017920:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.017924:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.017927:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810061696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.017944:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.017947:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.017949:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573086.017950:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.017952:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.017954:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.017956:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13552 00000400:00000200:0.0:1788573086.017958:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573086.017961:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.017963:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.017965:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017967:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.017969:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017970:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573086.017972:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.017973:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573086.017974:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.017976:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573086.017977:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573086.017978:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573086.017981:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017984:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.017987:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.017988:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573086.017998:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573086.018004:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018009:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.018012:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.018017:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.018021:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018023:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.018027:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573086.018033:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018036:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.018038:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573086.018042:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573086.018045:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.018048:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018050:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.018053:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.018055:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.018058:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573086.018060:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.018063:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810061824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.018071:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.018075:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.018081:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810061824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.018091:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.018097:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.018102:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573086.018103:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.018104:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.018106:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.018115:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13552 00000400:00000200:0.0:1788573086.018119:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573086.018124:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.018128:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.018131:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018135:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018139:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018140:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573086.018142:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.018144:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573086.018148:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.018149:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573086.018151:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573086.018152:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573086.018155:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018159:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018160:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018161:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573086.018162:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573086.018164:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018165:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.018166:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.018170:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.018174:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018178:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.018182:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573086.018185:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018189:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.018193:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573086.018195:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573086.018200:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.018210:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018216:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.018220:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.018223:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.018232:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573086.018239:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.018249:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810061952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.018262:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.018264:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.018269:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810061952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.018280:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.018283:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.018286:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573086.018289:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.018290:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.018294:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.018298:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13552 00000400:00000200:0.0:1788573086.018306:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573086.018311:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.018313:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.018318:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018322:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018327:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018332:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573086.018333:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.018334:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573086.018336:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.018338:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573086.018342:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573086.018351:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573086.018357:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018362:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018365:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018366:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573086.018368:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573086.018373:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018375:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.018377:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.018382:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.018386:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018389:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.018391:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573086.018394:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018398:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.018401:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573086.018404:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573086.018408:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.018410:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018414:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.018416:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.018418:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.018423:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573086.018427:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.018433:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810062080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.018444:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.018448:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.018452:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810062080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.018463:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.018470:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.018475:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573086.018477:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.018480:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.018484:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.018488:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13552 00000400:00000200:0.0:1788573086.018496:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573086.018500:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.018502:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.018505:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018508:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018512:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018517:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573086.018521:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.018523:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573086.018526:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.018529:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573086.018531:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573086.018532:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573086.018534:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018535:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.018537:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018539:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573086.018541:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573086.018544:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573086.018550:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.018555:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.018560:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.018565:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018570:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.018581:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573086.018588:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018592:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.018613:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573086.018618:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573086.018624:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.018625:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.018627:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.018631:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.018694:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.018703:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573086.018711:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.018722:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810062208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.018737:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.018742:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.018751:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810062208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.018771:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.018778:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13562) 00000020:00000001:0.0:1788573087.937852:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573087.937856:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573087.937859:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573089.857718:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308228830, at 4308228864 00000100:00000010:0.0:1788573089.857732:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788573093.057798:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573093.057801:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573093.057803:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573096.385775:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308235359, at 4308235392 00000100:00000010:0.0:1788573096.385809:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00080000:0.0:1788573097.793765:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.793774:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573097.793795:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.793798:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.793802:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.793808:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13564 00000400:00000200:0.0:1788573097.793816:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573097.793821:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.793824:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.793826:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793829:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.793831:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793832:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573097.793836:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.793838:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573097.793840:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.793842:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573097.793844:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573097.793845:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573097.793847:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793849:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.793850:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793851:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573097.793854:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573097.793856:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793858:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.793860:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.793862:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.793865:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.793866:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573097.793867:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.793874:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573097.793876:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573097.793883:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.793884:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.793887:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.793889:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.793892:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.793899:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573097.793905:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.793912:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810063744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.793920:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.793923:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.793927:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810063744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.793949:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.793951:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.793954:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573097.793955:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.793956:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.793958:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.793960:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13564 00000400:00000200:0.0:1788573097.793963:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573097.793965:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.793966:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.793968:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793970:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.793972:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793974:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573097.793975:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.793976:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573097.793978:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.793979:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573097.793985:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573097.793987:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573097.793990:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.793994:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.793998:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794002:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573097.794003:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573097.794008:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794013:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.794017:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.794023:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.794027:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794030:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.794034:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573097.794039:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794042:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.794045:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573097.794047:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573097.794049:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.794050:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794054:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.794056:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.794059:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.794064:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573097.794071:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.794078:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810063872/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.794089:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.794092:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.794099:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810063872/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.794121:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.794128:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.794133:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573097.794136:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.794139:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.794141:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.794163:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13564 00000400:00000200:0.0:1788573097.794170:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573097.794176:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.794180:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.794184:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794188:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794189:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794191:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573097.794196:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.794198:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573097.794199:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.794201:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573097.794204:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573097.794206:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573097.794212:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794213:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794214:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794216:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573097.794221:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573097.794227:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794229:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.794238:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.794245:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.794251:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794257:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.794262:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573097.794270:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794275:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.794283:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573097.794289:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573097.794296:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.794299:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794303:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.794305:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.794309:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.794317:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573097.794322:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.794333:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810064000/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.794349:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.794354:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.794361:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810064000/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.794371:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.794376:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.794381:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573097.794383:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.794388:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.794394:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.794398:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13564 00000400:00000200:0.0:1788573097.794403:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573097.794409:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.794414:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.794428:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794434:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794439:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794442:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573097.794444:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.794445:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573097.794449:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.794450:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573097.794453:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573097.794465:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573097.794469:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794471:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794473:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794474:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573097.794476:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573097.794482:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794486:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.794488:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.794493:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.794498:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794503:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.794506:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573097.794512:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794516:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.794522:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573097.794525:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573097.794529:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.794530:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794531:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.794532:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.794534:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.794539:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573097.794551:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.794559:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810064128/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.794567:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.794570:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.794578:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810064128/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.794590:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.794596:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.794601:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573097.794603:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.794607:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.794611:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.794617:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13564 00000400:00000200:0.0:1788573097.794669:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573097.794678:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.794684:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.794689:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794693:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794697:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794702:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573097.794707:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.794710:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573097.794712:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.794716:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573097.794718:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573097.794720:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573097.794722:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794723:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.794730:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794735:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573097.794737:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573097.794741:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573097.794743:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.794748:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.794752:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.794756:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794758:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.794762:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573097.794765:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794767:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.794773:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573097.794777:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573097.794783:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.794786:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.794791:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.794794:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.794798:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.794803:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573097.794808:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.794818:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810064256/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.794831:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.794835:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.794841:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810064256/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.794852:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.794858:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13574) 00000020:00000001:0.0:1788573098.177783:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573098.177788:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573098.177792:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573101.633783:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308240606, at 4308240640 00000100:00000010:0.0:1788573101.633804:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000100:00080000:0.0:1788573102.913701:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573102.913711:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573102.913713:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573102.913716:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573102.913719:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573102.913725:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13569 00000400:00000200:0.0:1788573102.913732:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573102.913738:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573102.913741:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573102.913743:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913745:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.913747:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913749:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573102.913752:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573102.913754:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573102.913756:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573102.913759:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573102.913761:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573102.913764:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573102.913767:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913769:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.913771:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913773:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573102.913776:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573102.913780:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913783:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573102.913786:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573102.913790:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573102.913811:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.913814:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573102.913816:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573102.913823:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573102.913826:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573102.913830:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573102.913832:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.913834:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573102.913836:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573102.913840:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573102.913846:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573102.913852:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.913859:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810064384/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573102.913867:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573102.913870:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.913873:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810064384/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573102.913892:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573102.913894:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573102.913897:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573102.913898:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573102.913899:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573102.913901:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573102.913902:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13569 00000400:00000200:0.0:1788573102.913905:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573102.913908:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573102.913909:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573102.913912:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913913:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.913915:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913921:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573102.913923:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573102.913924:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573102.913925:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573102.913927:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573102.913932:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573102.913934:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573102.913939:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913942:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.913959:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913964:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573102.913968:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573102.913975:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573102.913978:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573102.913981:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573102.913988:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573102.913990:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.913992:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573102.913997:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573102.914006:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914010:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573102.914018:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573102.914023:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573102.914030:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573102.914033:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914038:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573102.914042:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573102.914047:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573102.914053:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573102.914059:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.914064:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810064512/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573102.914092:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573102.914098:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.914110:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810064512/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573102.914131:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573102.914134:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573102.914137:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573102.914138:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573102.914139:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573102.914141:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573102.914144:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13569 00000400:00000200:0.0:1788573102.914152:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573102.914157:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573102.914162:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573102.914167:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914170:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914175:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914179:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573102.914184:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573102.914187:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573102.914190:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573102.914193:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573102.914196:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573102.914202:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573102.914205:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914210:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914216:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914220:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573102.914229:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573102.914236:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914240:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573102.914245:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573102.914250:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573102.914256:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914261:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573102.914263:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573102.914267:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914269:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573102.914273:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573102.914281:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573102.914286:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573102.914288:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914291:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573102.914294:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573102.914297:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573102.914304:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573102.914309:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.914319:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810064640/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573102.914329:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573102.914333:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.914339:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810064640/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573102.914354:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573102.914358:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573102.914363:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573102.914366:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573102.914369:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573102.914376:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573102.914388:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13569 00000400:00000200:0.0:1788573102.914397:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573102.914404:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573102.914407:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573102.914415:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914418:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914420:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914421:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573102.914422:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573102.914423:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573102.914427:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573102.914430:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573102.914432:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573102.914436:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573102.914441:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914445:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914448:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914452:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573102.914456:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573102.914460:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914464:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573102.914469:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573102.914473:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573102.914478:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914481:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573102.914486:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573102.914491:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914494:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573102.914500:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573102.914505:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573102.914518:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573102.914527:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914530:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573102.914533:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573102.914537:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573102.914544:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573102.914549:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.914557:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810064768/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573102.914571:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573102.914576:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.914585:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810064768/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573102.914603:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573102.914614:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573102.914620:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573102.914624:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573102.914628:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573102.914663:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573102.914670:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13569 00000400:00000200:0.0:1788573102.914679:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573102.914686:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573102.914689:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573102.914695:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914704:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914709:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914712:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573102.914716:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573102.914718:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573102.914721:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573102.914724:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573102.914730:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573102.914744:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573102.914752:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914759:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573102.914763:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914767:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573102.914770:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573102.914773:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573102.914778:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573102.914784:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573102.914790:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573102.914795:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914800:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573102.914818:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573102.914825:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914831:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573102.914839:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573102.914846:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573102.914852:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573102.914855:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.914861:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573102.914864:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573102.914869:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573102.914877:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573102.914886:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.914897:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810064896/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573102.914913:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573102.914918:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.914928:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810064896/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573102.914963:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573102.914976:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13579) 00000020:00000001:0.0:1788573103.297853:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573103.297858:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573103.297861:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573105.410082:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573106.753759:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308245726, at 4308245760 00000100:00000010:0.0:1788573106.753777:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788573108.417865:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573108.417868:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573108.417872:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573111.873778:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308250847, at 4308250880 00000100:00000010:0.0:1788573111.873798:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 10000000:00000001:0.0:1788573112.641724:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573112.641740:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e40. 00000020:00000010:0.0:1788573112.641749:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7000. 00000020:00000010:0.0:1788573112.641754:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7600. 00000040:00000010:0.0:1788573112.641758:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de6800. 00000020:00000010:0.0:1788573112.641763:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573112.641767:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573112.641771:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573112.641778:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3730c54c00. 20000000:00000010:0.0:1788573112.641782:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c55000. 10000000:00000040:0.0:1788573112.641787:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573112.641789:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573112.641805:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7600. 00000040:00000010:0.0:1788573112.641815:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de6800. 00000020:00000010:0.0:1788573112.641818:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573112.641821:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573112.641824:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573112.641828:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3730c54c00. 20000000:00000010:0.0:1788573112.641830:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c55000. 00000020:00000010:0.0:1788573112.641832:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7000. 10000000:00000010:0.0:1788573112.641835:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e40. 10000000:01000000:0.0:1788573112.641854:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573112.641857:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573112.641860:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573112.641861:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573112.641863:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573112.641867:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573112.641868:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573112.641871:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573112.641873:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573112.641875:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573112.641877:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573112.641878:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573112.641881:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788573113.537923:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573113.537931:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573113.537936:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573114.433738:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573114.433745:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573114.433750:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573114.433753:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573114.433755:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573114.433756:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573114.433759:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573114.433761:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573114.433764:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573114.433767:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573114.433768:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573114.433778:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573114.433782:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573114.433786:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573114.433789:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573114.433793:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573114.433800:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c55000. 00000100:00000001:0.0:1788573114.433806:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573114.433809:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573114.433812:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573114.433819:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46300. 00010000:00000001:0.0:1788573114.433839:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573114.433843:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788573114.433848:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573114.433851:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x1441586093168a73 to hash 00000020:00000001:0.0:1788573114.433854:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573114.433856:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573114.433859:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573114.433863:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573114.433866:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573114.433872:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573114.433880:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573114.433890:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573114.433894:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573114.433895:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573114.433901:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573114.433903:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573114.433905:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573114.433906:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788573114.433910:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573114.433914:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573114.433915:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573114.433917:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573114.433925:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573114.433932:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573114.433935:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573114.433951:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573114.433962:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573114.433968:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573114.433970:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573114.433971:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573114.433973:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573114.433975:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573114.433976:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573114.433979:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573114.433982:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573114.433983:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573114.433986:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573114.433992:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810066304:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573114.433995:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573114.433997:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573114.433999:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573114.434000:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573114.434003:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573114.434007:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573114.434009:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573114.434014:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573114.434015:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573114.434016:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573114.434018:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573114.434020:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573114.434022:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=4 00000020:00000001:0.0:1788573114.434027:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573114.434029:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00010000:0.0:1788573114.434031:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434037:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573114.434039:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434044:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573114.434046:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434050:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573114.434051:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573114.434053:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434059:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573114.434061:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573114.434070:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573114.434073:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573114.434081:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573114.434083:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573114.434086:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434093:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573114.434095:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573114.434096:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573114.434103:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=5 00000020:00000001:0.0:1788573114.434106:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573114.434108:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573114.434110:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573114.434112:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573114.434117:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573114.434119:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573114.434123:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573114.434125:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573114.434126:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573114.434128:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573114.434129:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573114.434131:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573114.434133:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573114.434134:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573114.434135:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573114.434137:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31400 with handle 0x1441586093168a73 from hash 00010000:00000001:0.0:1788573114.434138:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573114.434139:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573114.434140:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573114.434141:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573114.434142:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573114.434144:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573114.434145:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573114.434146:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573114.434147:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573114.434148:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573114.434150:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573114.434153:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573114.434157:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573114.434158:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573114.434160:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573114.434161:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573114.434162:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573114.434163:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573114.434164:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168a73 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573114.434169:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46300 count: 0 00010000:00000001:0.0:1788573114.434176:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573114.434177:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573114.434179:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573114.434182:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440810066304/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573114.434187:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573114.434189:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573114.434190:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573114.434192:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573114.434195:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c55000. 02000000:00000001:0.0:1788573114.434197:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573114.434199:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573114.434200:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788573114.434203:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573114.434204:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573114.434205:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573114.434207:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573114.434210:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573118.273775:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308257247, at 4308257280 00000100:00000010:0.0:1788573118.273799:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788573118.657892:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573118.657898:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573118.657902:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573123.649747:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308262623, at 4308262656 00000100:00000010:0.0:1788573123.649762:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573123.777782:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573123.777787:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573123.777791:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573128.769701:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308267742, at 4308267776 00000100:00000010:0.0:1788573128.769715:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788573128.897852:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573128.897856:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573128.897860:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573133.889744:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308272863, at 4308272896 00000100:00000010:0.0:1788573133.889776:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000020:00000001:0.0:1788573134.017810:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573134.017813:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573134.017816:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573134.657797:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573134.657811:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e40. 00000020:00000010:0.0:1788573134.657817:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c55000. 00000020:00000010:0.0:1788573134.657822:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c54c00. 00000040:00000010:0.0:1788573134.657825:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3730c54a00. 00000020:00000010:0.0:1788573134.657829:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573134.657833:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3730c54200. 80000000:00000010:0.0:1788573134.657837:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573134.657842:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788573134.657846:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7200. 10000000:00000040:0.0:1788573134.657850:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573134.657852:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573134.657856:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c54c00. 00000040:00000010:0.0:1788573134.657863:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3730c54a00. 00000020:00000010:0.0:1788573134.657865:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573134.657868:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3730c54200. 80000000:00000010:0.0:1788573134.657870:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573134.657872:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788573134.657883:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7200. 00000020:00000010:0.0:1788573134.657886:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c55000. 10000000:00000010:0.0:1788573134.657902:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e40. 10000000:01000000:0.0:1788573134.657905:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573134.657907:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573134.657909:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573134.657910:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573134.657912:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573134.657915:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573134.657916:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573134.657919:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573134.657920:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573134.657922:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573134.657924:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573134.657925:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573134.657927:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573136.449805:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573136.449813:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573136.449818:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573136.449821:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573136.449823:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573136.449824:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573136.449827:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573136.449829:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573136.449832:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573136.449835:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573136.449837:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573136.449850:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573136.449855:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573136.449860:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573136.449864:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573136.449870:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573136.449879:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7200. 00000100:00000001:0.0:1788573136.449886:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573136.449890:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573136.449894:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573136.449902:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46300. 00010000:00000001:0.0:1788573136.449906:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573136.449910:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30800. 00000020:00000001:0.0:1788573136.449938:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573136.449942:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30800 with handle 0x1441586093168a7a to hash 00000020:00000001:0.0:1788573136.449945:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573136.449947:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788573136.449954:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788573136.449959:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573136.449962:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573136.449966:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573136.449978:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573136.449991:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573136.449998:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573136.450000:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573136.450007:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573136.450010:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573136.450013:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573136.450014:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531988608 : -109714177563008 : ffff9c372d6b6680) 00000100:00000001:0.0:1788573136.450019:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573136.450024:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573136.450027:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573136.450029:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573136.450039:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573136.450049:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573136.450054:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573136.450062:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573136.450072:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573136.450081:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573136.450084:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573136.450086:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573136.450088:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573136.450091:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573136.450093:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573136.450097:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573136.450100:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573136.450102:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573136.450107:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573136.450116:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810069632:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573136.450122:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573136.450126:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573136.450128:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573136.450130:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573136.450134:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573136.450142:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573136.450145:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573136.450151:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573136.450153:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573136.450156:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573136.450159:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573136.450161:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573136.450164:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=4 00000020:00000001:0.0:1788573136.450166:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788573136.450170:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00010000:0.0:1788573136.450174:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450187:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573136.450192:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450199:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573136.450202:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450209:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573136.450210:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573136.450214:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450220:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573136.450223:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573136.450226:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573136.450229:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573136.450236:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573136.450239:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573136.450241:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450248:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573136.450249:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573136.450251:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573136.450253:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=5 00000020:00000001:0.0:1788573136.450255:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788573136.450258:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788573136.450268:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573136.450271:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573136.450278:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573136.450281:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573136.450288:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573136.450291:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573136.450293:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573136.450295:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573136.450297:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573136.450300:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573136.450302:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573136.450304:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573136.450306:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573136.450309:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30800 with handle 0x1441586093168a7a from hash 00010000:00000001:0.0:1788573136.450312:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573136.450314:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573136.450315:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573136.450316:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573136.450318:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573136.450319:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573136.450322:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573136.450323:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573136.450325:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573136.450327:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573136.450330:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573136.450337:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573136.450338:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573136.450339:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573136.450341:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573136.450348:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573136.450350:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573136.450351:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573136.450354:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168a7a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573136.450362:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46300 count: 0 00010000:00000001:0.0:1788573136.450371:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573136.450373:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573136.450376:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573136.450381:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6680 x1875440810069632/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573136.450389:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573136.450392:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573136.450393:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573136.450396:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573136.450399:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7200. 02000000:00000001:0.0:1788573136.450404:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573136.450406:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573136.450408:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6680. 00000100:00000001:0.0:1788573136.450411:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573136.450413:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573136.450415:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573136.450417:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573136.450421:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573139.137863:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573139.137870:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573139.137874:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573140.289768:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308279263, at 4308279296 00000100:00000010:0.0:1788573140.289785:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000800:00000200:0.0:1788573143.298040:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573144.257849:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573144.257853:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573144.257856:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573145.665777:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308284639, at 4308284672 00000100:00000010:0.0:1788573145.665818:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573149.377769:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573149.377772:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573149.377780:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573150.785752:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308289758, at 4308289792 00000100:00000010:0.0:1788573150.785767:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000100:00080000:0.0:1788573152.065790:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573152.065799:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573152.065818:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573152.065822:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573152.065826:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573152.065831:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13618 00000400:00000200:0.0:1788573152.065839:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573152.065845:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573152.065848:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573152.065850:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.065852:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.065854:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.065856:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573152.065859:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573152.065860:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573152.065863:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573152.065866:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573152.065867:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573152.065869:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573152.065871:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.065873:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.065874:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.065876:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573152.065878:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573152.065898:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573152.065915:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573152.065918:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573152.065921:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573152.065923:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.065925:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573152.065927:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573152.065932:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573152.065934:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573152.065937:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573152.065938:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.065941:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573152.065943:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573152.065947:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573152.065953:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573152.065958:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.065965:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810071680/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573152.065973:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573152.065976:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.065980:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810071680/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573152.065999:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573152.066002:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573152.066005:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573152.066006:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573152.066007:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573152.066009:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573152.066010:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13618 00000400:00000200:0.0:1788573152.066014:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573152.066017:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573152.066020:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573152.066030:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066032:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066035:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066038:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573152.066041:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573152.066045:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573152.066048:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573152.066052:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573152.066056:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573152.066060:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573152.066066:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066072:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066077:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066080:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573152.066084:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573152.066088:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066091:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573152.066096:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573152.066102:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573152.066108:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066110:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573152.066115:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573152.066119:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066138:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573152.066141:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573152.066143:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573152.066146:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573152.066149:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066151:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573152.066152:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573152.066154:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573152.066159:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573152.066171:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.066179:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810071808/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573152.066188:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573152.066191:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.066199:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810071808/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573152.066216:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573152.066221:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573152.066227:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573152.066231:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573152.066235:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573152.066241:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573152.066247:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13618 00000400:00000200:0.0:1788573152.066255:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573152.066263:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573152.066270:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573152.066276:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066281:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066286:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066290:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573152.066294:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573152.066298:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573152.066301:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573152.066304:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573152.066308:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573152.066311:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573152.066317:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066319:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066330:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066334:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573152.066340:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573152.066361:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066364:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573152.066366:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573152.066369:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573152.066374:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066376:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573152.066381:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573152.066383:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066388:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573152.066394:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573152.066397:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573152.066402:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573152.066403:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066405:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573152.066407:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573152.066410:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573152.066417:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573152.066423:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.066432:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810071936/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573152.066447:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573152.066450:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.066456:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810071936/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573152.066466:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573152.066468:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573152.066474:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573152.066476:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573152.066484:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573152.066489:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573152.066492:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13618 00000400:00000200:0.0:1788573152.066499:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573152.066520:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573152.066523:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573152.066528:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066532:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066536:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066540:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573152.066545:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573152.066548:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573152.066549:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573152.066551:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573152.066555:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573152.066558:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573152.066565:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066570:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066573:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066578:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573152.066584:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573152.066588:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066590:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573152.066592:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573152.066596:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573152.066601:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066605:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573152.066607:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573152.066612:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066614:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573152.066621:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573152.066624:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573152.066626:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573152.066627:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066630:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573152.066633:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573152.066634:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573152.066637:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573152.066640:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.066648:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810072064/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573152.066657:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573152.066660:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.066697:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810072064/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573152.066704:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573152.066706:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573152.066709:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573152.066713:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573152.066715:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573152.066720:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573152.066724:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13618 00000400:00000200:0.0:1788573152.066729:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573152.066734:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573152.066738:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573152.066744:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066748:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066751:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066755:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573152.066757:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573152.066765:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573152.066767:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573152.066770:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573152.066773:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573152.066777:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573152.066783:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066786:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573152.066790:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066794:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573152.066798:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573152.066802:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573152.066807:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573152.066809:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573152.066812:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573152.066815:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066817:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573152.066818:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573152.066821:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066825:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573152.066830:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573152.066833:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573152.066837:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573152.066841:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573152.066843:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573152.066845:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573152.066849:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573152.066854:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573152.066860:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.066869:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810072192/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573152.066880:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573152.066884:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.066897:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810072192/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573152.066912:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.066916:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13628) 00000020:00000001:0.0:1788573154.497838:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573154.497841:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573154.497845:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573155.905770:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308294878, at 4308294912 00000100:00000010:0.0:1788573155.905789:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 10000000:00000001:0.0:1788573156.673700:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573156.673712:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e80. 00000020:00000010:0.0:1788573156.673719:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7200. 00000020:00000010:0.0:1788573156.673724:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7000. 00000040:00000010:0.0:1788573156.673727:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de6800. 00000020:00000010:0.0:1788573156.673730:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573156.673734:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573156.673738:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573156.673743:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3730c55000. 20000000:00000010:0.0:1788573156.673747:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c54200. 10000000:00000040:0.0:1788573156.673751:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573156.673753:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573156.673757:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7000. 00000040:00000010:0.0:1788573156.673765:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de6800. 00000020:00000010:0.0:1788573156.673767:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573156.673770:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573156.673772:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573156.673774:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3730c55000. 20000000:00000010:0.0:1788573156.673776:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c54200. 00000020:00000010:0.0:1788573156.673778:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7200. 10000000:00000010:0.0:1788573156.673780:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e80. 10000000:01000000:0.0:1788573156.673782:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573156.673785:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573156.673800:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573156.673801:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573156.673803:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573156.673805:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573156.673807:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573156.673809:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573156.673810:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573156.673812:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573156.673814:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573156.673816:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573156.673818:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788573157.185755:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573157.185764:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573157.185767:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573157.185771:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573157.185774:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573157.185780:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13623 00000400:00000200:0.0:1788573157.185788:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573157.185794:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573157.185797:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573157.185799:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.185801:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.185804:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.185805:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573157.185809:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573157.185810:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573157.185812:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573157.185815:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573157.185816:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573157.185818:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573157.185820:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.185822:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.185841:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.185842:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573157.185845:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573157.185848:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573157.185850:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573157.185852:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573157.185855:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573157.185857:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.185858:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573157.185860:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573157.185865:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573157.185869:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573157.185872:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573157.185873:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.185876:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573157.185894:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573157.185897:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573157.185903:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573157.185909:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573157.185916:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810072320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573157.185925:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573157.185929:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.185932:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810072320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573157.185966:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573157.185969:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573157.185972:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573157.185973:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573157.185974:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573157.185976:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573157.185989:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13623 00000400:00000200:0.0:1788573157.185994:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573157.185998:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573157.186001:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573157.186005:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186008:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186012:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186018:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573157.186023:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573157.186027:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573157.186031:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573157.186035:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573157.186041:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573157.186048:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573157.186056:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186062:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186068:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186074:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573157.186080:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573157.186087:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186091:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573157.186096:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573157.186104:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573157.186110:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186114:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573157.186120:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573157.186126:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186130:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573157.186134:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573157.186139:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573157.186142:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573157.186150:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186152:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573157.186154:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573157.186157:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573157.186163:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573157.186168:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573157.186176:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810072448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573157.186205:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573157.186210:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.186218:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810072448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573157.186232:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573157.186236:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573157.186242:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573157.186243:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573157.186245:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573157.186249:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573157.186253:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13623 00000400:00000200:0.0:1788573157.186261:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573157.186268:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573157.186270:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573157.186276:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186280:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186284:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186287:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573157.186291:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573157.186294:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573157.186296:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573157.186299:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573157.186302:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573157.186311:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573157.186319:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186323:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186328:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186330:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573157.186332:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573157.186337:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186342:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573157.186345:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573157.186349:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573157.186352:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186356:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573157.186358:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573157.186362:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186363:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573157.186367:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573157.186370:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573157.186373:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573157.186375:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186377:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573157.186379:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573157.186381:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573157.186384:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573157.186391:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573157.186396:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810072576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573157.186425:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573157.186428:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.186436:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810072576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573157.186448:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573157.186457:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573157.186462:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573157.186466:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573157.186476:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573157.186480:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573157.186485:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13623 00000400:00000200:0.0:1788573157.186491:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573157.186498:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573157.186503:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573157.186506:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186508:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186509:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186511:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573157.186512:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573157.186513:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573157.186514:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573157.186515:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573157.186518:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573157.186521:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573157.186525:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186529:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186533:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186538:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573157.186542:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573157.186547:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186553:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573157.186555:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573157.186560:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573157.186563:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186565:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573157.186574:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573157.186579:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186582:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573157.186587:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573157.186592:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573157.186594:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573157.186596:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186598:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573157.186600:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573157.186602:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573157.186606:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573157.186609:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573157.186616:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810072704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573157.186632:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573157.186684:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.186694:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810072704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573157.186720:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573157.186724:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573157.186727:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573157.186728:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573157.186731:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573157.186735:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573157.186740:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13623 00000400:00000200:0.0:1788573157.186745:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573157.186750:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573157.186754:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573157.186759:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186764:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186768:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186778:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573157.186784:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573157.186787:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573157.186790:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573157.186793:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573157.186797:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573157.186800:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573157.186807:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186810:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573157.186814:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186817:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573157.186821:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573157.186826:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573157.186831:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573157.186835:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573157.186840:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573157.186844:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186847:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573157.186850:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573157.186852:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186857:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573157.186880:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573157.186884:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573157.186888:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573157.186891:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573157.186897:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573157.186898:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573157.186902:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573157.186907:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573157.186911:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573157.186919:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810072832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573157.186978:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573157.186982:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.186992:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810072832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573157.187007:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.187013:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13633) 10000000:00000001:0.0:1788573158.593750:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573158.593757:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573158.593761:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573158.593763:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573158.593765:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573158.593766:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573158.593769:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573158.593771:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573158.593773:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573158.593776:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573158.593778:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573158.593790:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573158.593795:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573158.593799:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.593802:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.593808:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.593817:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c54200. 00000100:00000001:0.0:1788573158.593823:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573158.593827:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573158.593830:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573158.593839:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46c00. 00010000:00000001:0.0:1788573158.593843:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573158.593847:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30a00. 00000020:00000001:0.0:1788573158.593851:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573158.593854:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30a00 with handle 0x1441586093168a81 to hash 00000020:00000001:0.0:1788573158.593855:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573158.593856:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573158.593859:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573158.593875:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573158.593878:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573158.593882:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573158.593890:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573158.593914:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573158.593918:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573158.593920:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573158.593925:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573158.593926:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573158.593928:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573158.593929:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788573158.593933:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.593937:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.593938:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573158.593940:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573158.593948:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.593954:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573158.593957:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.593962:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.593965:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.593969:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573158.593971:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573158.593976:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.593978:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573158.593980:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573158.593982:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573158.593984:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.593986:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573158.593988:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573158.593991:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573158.593997:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810072960:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573158.594000:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.594002:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573158.594004:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573158.594006:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573158.594008:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.594013:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.594016:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573158.594020:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573158.594021:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573158.594023:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573158.594025:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573158.594027:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.594029:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=4 00000020:00000001:0.0:1788573158.594030:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573158.594032:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00010000:0.0:1788573158.594035:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594040:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573158.594044:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594055:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573158.594058:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594065:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573158.594067:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573158.594069:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594075:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573158.594077:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573158.594080:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573158.594083:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573158.594090:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573158.594093:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573158.594096:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594102:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573158.594104:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573158.594105:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.594107:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788573158.594108:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573158.594110:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573158.594112:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573158.594114:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573158.594118:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573158.594124:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573158.594129:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573158.594130:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573158.594131:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573158.594134:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573158.594135:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573158.594137:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573158.594138:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573158.594139:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573158.594141:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573158.594142:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30a00 with handle 0x1441586093168a81 from hash 00010000:00000001:0.0:1788573158.594144:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573158.594145:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573158.594146:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573158.594147:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573158.594148:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573158.594149:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573158.594151:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573158.594151:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573158.594152:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573158.594153:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573158.594155:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573158.594159:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573158.594159:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573158.594160:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573158.594161:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573158.594163:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573158.594163:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573158.594164:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573158.594165:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168a81 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573158.594170:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46c00 count: 0 00010000:00000001:0.0:1788573158.594180:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573158.594181:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573158.594183:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573158.594187:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810072960/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.594192:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573158.594193:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573158.594195:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573158.594196:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573158.594199:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c54200. 02000000:00000001:0.0:1788573158.594201:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573158.594203:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573158.594204:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788573158.594207:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573158.594208:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573158.594209:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573158.594211:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573158.594213:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573159.617943:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573159.617947:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573159.617950:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573162.433721:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308301407, at 4308301440 00000100:00000010:0.0:1788573162.433736:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000100:00000001:0.0:1788573163.841978:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573163.841984:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573163.841988:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573163.841999:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842018:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.842022:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.842031:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573163.842037:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.842045:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.842065:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810073728:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573163.842088:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573163.842094:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573163.842098:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.842103:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573163.842109:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573163.842114:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.842124:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573163.842128:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573163.842137:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573163.842146:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810073728, portal 25 00000100:00000001:0.0:1788573163.842151:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573163.842155:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573163.842164:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842182:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573163.842190:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573163.842194:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573163.842198:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810073728, offset 0 00000400:00000200:0.0:1788573163.842208:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573163.842221:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.842232:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573163.842241:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573163.842246:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573163.842251:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573163.842256:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573163.842263:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540696:540696:256:4294967295] 192.168.202.148@tcp LPNI seq info [540661:540661:8:4294967295] 00000400:00000200:0.0:1788573163.842276:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.842301:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573163.842315:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.842326:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573163.842343:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573163.842359:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573163.842393:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.842399:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573163.842409:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573163.842423:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573163.842433:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573163.842438:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573163.842446:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573163.842454:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842473:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.842477:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842487:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.842492:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573163.842501:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.842510:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573163.842515:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308306654 00000100:00000010:0.0:1788573163.842526:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573163.842535:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573163.842543:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308306654 00000100:00000010:0.0:1788573163.842551:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626f00. 00000100:00000001:0.0:1788573163.842558:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573163.842564:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573163.842568:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.842571:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573163.842581:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573163/real 1788573163] req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573163.842599:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573163.842602:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573163.842611:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573163.842615:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573163.842618:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573163.842623:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573163.842630:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573163.842663:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842672:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573163.842675:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842678:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.842682:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573163.842689:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842700:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573163.842707:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842734:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573163.842740:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842744:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.842747:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842750:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573163.842754:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573163.842759:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573163.842765:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573163.842778:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573163.842786:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573163.842800:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810073728:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573163.842819:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842822:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.842831:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810073728/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842840:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573163.842843:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573163.842846:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573163.842849:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573163.842853:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573163.842856:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573163.842861:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573163.842864:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573163.842869:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573163.842872:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573163.842875:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573163.842879:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573163.842882:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842884:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.842890:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573163.842894:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.842897:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573163.842902:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573163.842905:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573163.842913:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.842927:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.842930:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.842934:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573163.842939:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.842944:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.842953:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810073856:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573163.842962:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573163.842967:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573163.842971:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.842981:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573163.842985:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573163.842990:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.842997:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573163.843004:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573163.843008:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573163.843014:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810073856, portal 10 00000100:00000001:0.0:1788573163.843016:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573163.843017:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573163.843025:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843033:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573163.843036:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573163.843038:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573163.843041:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810073856, offset 0 00000400:00000200:0.0:1788573163.843045:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843050:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843054:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573163.843058:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573163.843060:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573163.843063:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573163.843066:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843069:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540697:540697:256:4294967295] 192.168.202.148@tcp LPNI seq info [540662:540662:8:4294967295] 00000400:00000200:0.0:1788573163.843074:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843081:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573163.843084:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843087:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573163.843091:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573163.843093:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573163.843101:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843102:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573163.843109:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573163.843111:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573163.843114:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573163.843116:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573163.843118:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573163.843120:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843126:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843129:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843133:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843134:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573163.843137:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.843140:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573163.843142:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573163.843144:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308302873 00000100:00000001:0.0:1788573163.843146:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843148:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308302873 00000100:00000001:0.0:1788573163.843150:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843151:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573163.843153:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843154:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573163.843156:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573163/real 1788573163] req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573163.843162:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573163.843163:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573163.843164:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573163.843166:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573163.843167:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573163.843168:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573163.843174:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573163.843179:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843184:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573163.843185:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843186:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843187:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573163.843189:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843194:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573163.843196:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843201:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573163.843202:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843203:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843204:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843205:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573163.843206:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573163.843208:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573163.843210:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573163.843211:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573163.843213:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573163.843218:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810073856:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573163.843221:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573163.843223:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843234:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.843235:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843237:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573163.843238:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.843240:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.843241:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810073984:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573163.843244:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573163.843245:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573163.843246:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843248:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843250:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573163.843251:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.843254:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573163.843255:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573163.843258:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573163.843260:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810073984, portal 10 00000100:00000001:0.0:1788573163.843261:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573163.843263:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573163.843267:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843271:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573163.843273:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573163.843274:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573163.843276:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810073984, offset 0 00000400:00000200:0.0:1788573163.843278:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843281:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843283:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573163.843285:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573163.843287:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573163.843288:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573163.843290:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843295:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540698:540698:256:4294967295] 192.168.202.148@tcp LPNI seq info [540663:540663:8:4294967295] 00000400:00000200:0.0:1788573163.843298:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843303:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573163.843306:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843309:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573163.843311:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573163.843314:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573163.843321:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843322:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573163.843324:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573163.843327:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573163.843329:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573163.843330:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573163.843332:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573163.843334:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843340:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843342:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843346:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843347:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573163.843349:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.843352:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573163.843354:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573163.843355:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308302877 00000100:00000001:0.0:1788573163.843357:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843359:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308302877 00000100:00000001:0.0:1788573163.843360:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843362:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573163.843366:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843368:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573163.843371:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573163/real 1788573163] req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573163.843377:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573163.843378:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573163.843379:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573163.843380:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573163.843381:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573163.843383:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573163.843385:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573163.843390:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843394:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573163.843396:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843397:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573163.843399:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843403:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573163.843406:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843410:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573163.843411:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843412:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843413:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843414:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573163.843415:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573163.843416:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573163.843423:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573163.843424:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573163.843427:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573163.843432:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810073984:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573163.843434:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573163.843436:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843440:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.843441:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843442:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573163.843444:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.843445:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.843447:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074112:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573163.843449:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573163.843450:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573163.843451:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843453:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843454:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573163.843456:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.843458:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573163.843459:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573163.843462:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573163.843464:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074112, portal 4 00000100:00000001:0.0:1788573163.843465:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573163.843466:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573163.843470:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843475:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573163.843479:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573163.843480:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573163.843482:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810074112, offset 0 00000400:00000200:0.0:1788573163.843485:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843488:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843490:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573163.843492:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573163.843494:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573163.843496:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573163.843497:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843500:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540699:540699:256:4294967295] 192.168.202.148@tcp LPNI seq info [540664:540664:8:4294967295] 00000400:00000200:0.0:1788573163.843503:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843506:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573163.843509:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843511:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573163.843513:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573163.843515:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573163.843522:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843524:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573163.843526:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573163.843528:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573163.843530:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573163.843531:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573163.843533:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573163.843535:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843540:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843542:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843546:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843547:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573163.843553:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.843555:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573163.843557:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573163.843559:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308303009 00000100:00000001:0.0:1788573163.843560:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843562:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308303009 00000100:00000001:0.0:1788573163.843563:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843564:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573163.843566:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843567:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573163.843569:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573163/real 1788573163] req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573163.843576:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573163.843578:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573163.843579:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573163.843582:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573163.843584:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573163.843586:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573163.843589:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573163.843598:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843606:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573163.843608:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843610:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843611:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573163.843615:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843621:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573163.843625:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843658:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573163.843661:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843663:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843664:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843666:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573163.843668:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573163.843669:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573163.843672:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573163.843674:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573163.843677:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573163.843682:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074112:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573163.843685:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843686:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843688:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810073856/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843693:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573163.843694:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843696:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573163.843698:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573163.843699:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573163.843700:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573163.843701:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573163.843703:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573163.843705:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573163.843706:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573163.843707:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573163.843709:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573163.843710:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843729:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843732:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810073984/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843736:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573163.843737:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843738:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573163.843740:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573163.843741:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573163.843741:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573163.843743:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573163.843744:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573163.843745:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573163.843746:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573163.843747:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573163.843749:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573163.843749:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843750:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843752:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810074112/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843756:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573163.843757:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843758:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573163.843759:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573163.843760:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573163.843761:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573163.843762:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573163.843762:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573163.843764:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573163.843764:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573163.843765:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573163.843766:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573163.843767:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843768:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843771:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573163.843772:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843773:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573163.843776:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573163.843778:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573163.843780:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843785:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.843786:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843788:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573163.843789:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.843790:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.843792:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074240:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573163.843794:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573163.843795:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573163.843796:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.843797:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573163.843799:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573163.843800:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573163.843803:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573163.843805:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573163.843807:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573163.843810:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074240, portal 4 00000100:00000001:0.0:1788573163.843811:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573163.843812:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573163.843817:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843821:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573163.843822:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573163.843824:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573163.843825:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810074240, offset 0 00000400:00000200:0.0:1788573163.843828:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843830:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843833:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573163.843835:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573163.843836:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573163.843841:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573163.843843:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573163.843845:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540700:540700:256:4294967295] 192.168.202.148@tcp LPNI seq info [540665:540665:8:4294967295] 00000400:00000200:0.0:1788573163.843848:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573163.843852:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573163.843855:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843857:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573163.843860:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573163.843861:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573163.843873:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573163.843874:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573163.843876:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573163.843878:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573163.843881:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573163.843882:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573163.843884:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573163.843887:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843892:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.843894:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573188 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843899:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843900:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573163.843903:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.843906:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573163.843910:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573163.843912:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308303019 00000100:00000001:0.0:1788573163.843914:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843917:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308303019 00000100:00000001:0.0:1788573163.843925:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308306654 00000100:00000001:0.0:1788573163.843927:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573163.843930:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.843932:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573163.843936:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573163/real 1788573163] req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573163.843946:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573163.843947:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573163.843949:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8ec8. 00000400:00000200:0.0:1788573163.843952:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed314c8 00000400:00000010:0.0:1788573163.843954:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed314c8. 00000100:00000001:0.0:1788573163.843956:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573163.843959:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573163.843965:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843969:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573163.843970:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843971:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843972:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573163.843974:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843979:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573163.843981:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.843985:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573163.843987:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843988:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573163.843988:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.843994:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573163.843995:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573163.843997:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573163.844000:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573163.844001:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573163.844004:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573163.844009:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074240:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573163.844012:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.844013:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573163.844015:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810074240/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573188 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.844019:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573163.844020:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573163.844021:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573163.844022:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573163.844024:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573163.844025:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573163.844026:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573163.844028:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573163.844030:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573163.844032:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573163.844033:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573163.844036:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573163.844037:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.844039:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.844042:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573163.844044:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.844046:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573163.844048:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573164.737772:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573164.737775:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573164.737791:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573167.681703:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308306654, at 4308306688 00000100:00000010:0.0:1788573167.681718:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573168.961981:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573168.961989:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573168.961994:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573168.962007:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72d80 x1875440810074368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962029:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.962033:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962041:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573168.962047:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573168.962056:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.962065:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810074368:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573168.962076:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573168.962082:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573168.962088:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962094:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573168.962102:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573168.962108:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.962122:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573168.962130:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8ec8. 00000400:00000010:0.0:1788573168.962140:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed314c8. 00000100:00000200:0.0:1788573168.962152:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074368, portal 25 00000100:00000001:0.0:1788573168.962158:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573168.962163:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345088 : -109714423206528 : ffff9c371ec72d80) 00000100:00000040:0.0:1788573168.962174:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72d80 x1875440810074368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962200:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573168.962209:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573168.962216:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573168.962223:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810074368, offset 0 00000400:00000200:0.0:1788573168.962240:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962263:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962284:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573168.962293:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573168.962304:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573168.962315:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573168.962325:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962338:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540701:540701:256:4294967295] 192.168.202.148@tcp LPNI seq info [540666:540666:8:4294967295] 00000400:00000200:0.0:1788573168.962355:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962378:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573168.962394:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573168.962405:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573168.962419:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573168.962458:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.962468:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573168.962475:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308311774 00000100:00000010:0.0:1788573168.962484:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573168.962496:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573168.962506:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308311774 00000100:00000010:0.0:1788573168.962515:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626720. 00000100:00000001:0.0:1788573168.962523:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573168.962547:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573168.962551:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.962555:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.962557:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573168.962566:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573168.962568:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573168.962570:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573168.962572:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573168.962573:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573168.962587:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810074496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962608:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.962619:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962622:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573168.962627:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573168.962671:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.962682:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074496:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573168.962690:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573168.962694:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573168.962699:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962704:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573168.962710:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573168.962715:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.962723:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573168.962728:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8580. 00000400:00000010:0.0:1788573168.962734:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31cc0. 00000100:00000200:0.0:1788573168.962738:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074496, portal 10 00000100:00000001:0.0:1788573168.962740:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573168.962741:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573168.962750:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810074496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962758:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573168.962761:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573168.962763:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788573168.962766:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810074496, offset 0 00000400:00000200:0.0:1788573168.962771:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962775:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962780:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573168.962783:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573168.962786:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573168.962789:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573168.962791:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962794:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540702:540702:255:4294967295] 192.168.202.148@tcp LPNI seq info [540667:540667:7:4294967295] 00000400:00000200:0.0:1788573168.962799:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962812:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573168.962823:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573168.962827:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788573168.962831:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573168.962840:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573168.962844:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573168.962847:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573168.962849:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308307993 00000100:00000001:0.0:1788573168.962852:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.962854:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308307993 00000100:00000001:0.0:1788573168.962857:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.962859:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573168.962861:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.962863:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573168.962868:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810074624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962875:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.962877:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962879:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573168.962881:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573168.962884:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.962887:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074624:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573168.962891:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573168.962893:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573168.962894:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.962896:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573168.962899:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573168.962902:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.962905:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573168.962908:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8000. 00000400:00000010:0.0:1788573168.962912:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31dd0. 00000100:00000200:0.0:1788573168.962925:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074624, portal 10 00000100:00000001:0.0:1788573168.962927:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573168.962928:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573168.962935:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810074624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.962942:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573168.962944:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573168.962947:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788573168.962949:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810074624, offset 0 00000400:00000200:0.0:1788573168.962953:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962957:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962961:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573168.962964:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573168.962966:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573168.962969:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573168.962971:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573168.962974:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540703:540703:254:4294967295] 192.168.202.148@tcp LPNI seq info [540668:540668:6:4294967295] 00000400:00000200:0.0:1788573168.962978:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.962984:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573168.962989:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573168.962993:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573168.962997:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573168.963004:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573168.963007:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573168.963011:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573168.963013:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308307997 00000100:00000001:0.0:1788573168.963015:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963017:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308307997 00000100:00000001:0.0:1788573168.963020:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963022:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573168.963029:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963031:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573168.963035:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810074752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.963043:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.963045:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.963047:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573168.963049:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573168.963051:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.963054:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074752:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573168.963058:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573168.963060:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573168.963061:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.963063:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573168.963066:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573168.963068:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.963072:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573168.963075:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8e18. 00000400:00000010:0.0:1788573168.963079:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31f68. 00000100:00000200:0.0:1788573168.963082:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074752, portal 4 00000100:00000001:0.0:1788573168.963084:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573168.963085:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573168.963091:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810074752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.963098:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573168.963100:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573168.963102:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573168.963105:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810074752, offset 0 00000400:00000200:0.0:1788573168.963109:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573168.963113:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.963116:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573168.963119:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573168.963126:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573168.963129:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573168.963131:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573168.963135:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540704:540704:253:4294967295] 192.168.202.148@tcp LPNI seq info [540669:540669:5:4294967295] 00000400:00000200:0.0:1788573168.963139:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.963146:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573168.963151:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573168.963154:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788573168.963158:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573168.963165:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573168.963168:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573168.963172:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573168.963174:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308308129 00000100:00000001:0.0:1788573168.963182:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963184:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308308129 00000100:00000001:0.0:1788573168.963187:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963189:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573168.963192:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963194:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963197:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573168.963200:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573168.963202:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573168.963204:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573168.963205:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573168.963207:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573168.963211:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810074880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.963219:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.963220:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.963222:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573168.963224:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573168.963230:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.963234:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810074880:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573168.963237:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573168.963239:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573168.963241:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573168.963243:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573168.963245:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573168.963247:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573168.963251:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573168.963253:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8898. 00000400:00000010:0.0:1788573168.963258:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31bb0. 00000100:00000200:0.0:1788573168.963261:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810074880, portal 4 00000100:00000001:0.0:1788573168.963263:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573168.963265:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573168.963272:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810074880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573168.963278:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573168.963281:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573168.963283:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573168.963286:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810074880, offset 0 00000400:00000200:0.0:1788573168.963290:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573168.963293:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.963297:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573168.963300:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573168.963302:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573168.963304:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573168.963307:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573168.963310:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540705:540705:252:4294967295] 192.168.202.148@tcp LPNI seq info [540670:540670:4:4294967295] 00000400:00000200:0.0:1788573168.963314:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573168.963321:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573168.963325:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573168.963332:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573168.963336:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573168.963343:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573168.963346:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573168.963349:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573168.963351:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308308138 00000100:00000001:0.0:1788573168.963354:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963356:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308308138 00000100:00000001:0.0:1788573168.963358:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308311774 00000100:00000001:0.0:1788573168.963361:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573168.963363:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963366:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963368:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573168.963371:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573168.963373:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573168.963375:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573168.963376:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573168.963378:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573168.963381:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573169.857889:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573169.857892:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573169.857895:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788573172.033955:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788573172.034024:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573172.034038:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573172.034043:0:2390:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788573172.034050:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573172.034055:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573172.034063:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573172.034069:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0e00. 00000400:00000200:0.0:1788573172.034078:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573172.034085:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573172.034118:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573172.034122:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573172.034129:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573172.034141:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec72d80 x1875440810074368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034186:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573172.034192:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72d80 x1875440810074368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034202:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573172.034204:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573172.034216:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573172.034219:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0000. 00000400:00000200:0.0:1788573172.034228:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573172.034232:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573172.034237:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31110 00000400:00000010:0.0:1788573172.034239:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31110. 00000100:00000001:0.0:1788573172.034242:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573172.034248:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70380 x1875440810074496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034257:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573172.034260:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810074496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034267:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573172.034269:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573172.034274:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573172.034277:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573172.034280:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573172.034283:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573172.034286:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31aa0 00000400:00000010:0.0:1788573172.034288:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31aa0. 00000100:00000001:0.0:1788573172.034291:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573172.034295:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec73480 x1875440810074624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034312:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573172.034315:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810074624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034322:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573172.034323:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573172.034330:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573172.034333:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788573172.034336:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573172.034344:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573172.034347:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573172.034349:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573172.034352:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573172.034356:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70e00 x1875440810074752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034363:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573172.034366:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810074752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034372:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573172.034374:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573172.034379:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573172.034381:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573172.034384:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573172.034387:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573172.034390:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573172.034392:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573172.034394:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573172.034399:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70000 x1875440810074880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034406:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573172.034408:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810074880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573172.034441:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573172.034443:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788573172.801756:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308311774, at 4308311808 00000100:00000010:0.0:1788573172.801770:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00080000:0.0:1788573174.081722:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573174.081731:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573174.081733:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573174.081736:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573174.081740:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573174.081747:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13640 00000400:00000200:0.0:1788573174.081754:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573174.081759:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573174.081762:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573174.081764:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081766:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.081768:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081770:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573174.081773:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573174.081774:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573174.081777:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573174.081780:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573174.081781:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573174.081783:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573174.081785:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081787:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.081788:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081789:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573174.081791:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573174.081794:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081809:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573174.081811:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573174.081814:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573174.081816:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.081818:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573174.081819:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573174.081824:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573174.081828:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573174.081832:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573174.081834:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.081837:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573174.081840:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573174.081844:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573174.081852:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573174.081858:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.081868:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810075008/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573174.081879:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573174.081883:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.081887:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810075008/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573174.081915:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573174.081919:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573174.081922:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573174.081924:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573174.081926:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573174.081929:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573174.081932:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13640 00000400:00000200:0.0:1788573174.081938:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573174.081942:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573174.081945:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573174.081949:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081958:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.081976:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.081980:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573174.081985:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573174.081988:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573174.081992:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573174.082022:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573174.082028:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573174.082035:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573174.082043:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082050:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082057:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082061:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573174.082065:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573174.082068:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082072:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573174.082074:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573174.082083:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573174.082088:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082094:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573174.082101:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573174.082106:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082109:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573174.082118:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573174.082122:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573174.082131:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573174.082135:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082140:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573174.082142:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573174.082146:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573174.082153:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573174.082160:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.082176:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810075136/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573174.082191:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573174.082195:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.082203:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810075136/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573174.082216:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573174.082220:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573174.082227:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573174.082231:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573174.082236:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573174.082241:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573174.082249:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13640 00000400:00000200:0.0:1788573174.082255:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573174.082262:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573174.082266:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573174.082276:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082282:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082285:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082287:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573174.082290:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573174.082291:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573174.082294:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573174.082299:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573174.082303:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573174.082309:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573174.082315:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082320:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082322:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082333:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573174.082340:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573174.082348:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082369:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573174.082374:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573174.082379:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573174.082385:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082390:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573174.082395:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573174.082400:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082405:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573174.082411:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573174.082415:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573174.082419:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573174.082422:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082425:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573174.082428:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573174.082432:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573174.082447:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573174.082454:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.082465:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810075264/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573174.082478:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573174.082483:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.082493:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810075264/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573174.082505:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573174.082508:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573174.082511:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573174.082514:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573174.082517:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573174.082530:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573174.082536:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13640 00000400:00000200:0.0:1788573174.082546:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573174.082551:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573174.082556:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573174.082560:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082566:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082571:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082576:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573174.082581:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573174.082586:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573174.082591:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573174.082596:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573174.082601:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573174.082606:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573174.082611:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082614:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082616:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082617:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573174.082620:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573174.082622:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082625:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573174.082627:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573174.082630:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573174.082632:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082635:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573174.082668:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573174.082671:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082672:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573174.082677:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573174.082686:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573174.082690:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573174.082692:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082694:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573174.082695:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573174.082698:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573174.082702:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573174.082707:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.082714:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810075392/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573174.082722:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573174.082725:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.082729:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810075392/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573174.082760:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573174.082764:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573174.082771:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573174.082775:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573174.082779:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573174.082784:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573174.082791:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13640 00000400:00000200:0.0:1788573174.082800:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573174.082808:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573174.082812:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573174.082816:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082822:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082826:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082831:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573174.082836:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573174.082837:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573174.082852:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573174.082857:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573174.082863:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573174.082868:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573174.082878:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082884:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573174.082889:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082894:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573174.082900:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573174.082907:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573174.082914:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573174.082919:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573174.082924:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573174.082930:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082936:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573174.082942:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573174.082948:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082952:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573174.082961:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573174.082968:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573174.082974:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573174.082978:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.082981:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573174.082984:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573174.082988:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573174.082996:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573174.083002:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.083014:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810075520/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573174.083030:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573174.083036:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.083048:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810075520/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573174.083068:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573174.083076:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13650) 00000020:00000001:0.0:1788573174.977794:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573174.977797:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573174.977800:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573177.921755:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308316894, at 4308316928 00000100:00000010:0.0:1788573177.921774:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000100:00080000:0.0:1788573179.201768:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573179.201780:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573179.201782:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573179.201786:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573179.201790:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573179.201798:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13645 00000400:00000200:0.0:1788573179.201808:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573179.201815:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573179.201818:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573179.201821:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.201824:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.201826:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.201828:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573179.201831:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573179.201833:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573179.201836:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573179.201839:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573179.201840:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573179.201842:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573179.201844:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.201846:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.201847:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.201885:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573179.201888:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573179.201891:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573179.201893:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573179.201896:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573179.201899:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573179.201902:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.201904:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573179.201906:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573179.201915:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573179.201918:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573179.201922:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573179.201924:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.201927:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573179.201930:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573179.201934:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573179.201942:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573179.201958:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.201972:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810075648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573179.201985:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573179.201989:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.201994:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810075648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573179.202015:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573179.202018:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573179.202021:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573179.202022:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573179.202024:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573179.202026:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573179.202027:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13645 00000400:00000200:0.0:1788573179.202035:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573179.202038:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573179.202040:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573179.202042:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202044:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202045:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202047:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573179.202049:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573179.202050:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573179.202051:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573179.202053:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573179.202055:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573179.202057:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573179.202059:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202060:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202062:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202063:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573179.202065:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573179.202066:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202068:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573179.202070:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573179.202077:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573179.202079:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202080:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573179.202091:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573179.202096:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202098:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573179.202102:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573179.202104:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573179.202110:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573179.202116:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202119:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573179.202126:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573179.202128:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573179.202133:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573179.202137:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.202144:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810075776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573179.202174:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573179.202187:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.202191:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810075776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573179.202206:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573179.202211:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573179.202223:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573179.202226:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573179.202228:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573179.202231:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573179.202234:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13645 00000400:00000200:0.0:1788573179.202245:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573179.202249:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573179.202254:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573179.202260:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202265:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202268:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202269:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573179.202271:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573179.202272:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573179.202273:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573179.202274:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573179.202275:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573179.202286:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573179.202293:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202296:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202302:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202304:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573179.202306:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573179.202309:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202314:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573179.202322:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573179.202326:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573179.202330:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202331:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573179.202333:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573179.202337:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202344:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573179.202355:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573179.202358:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573179.202363:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573179.202364:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202366:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573179.202367:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573179.202368:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573179.202372:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573179.202374:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.202378:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810075904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573179.202396:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573179.202398:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.202402:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810075904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573179.202411:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573179.202420:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573179.202425:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573179.202428:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573179.202434:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573179.202436:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573179.202440:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13645 00000400:00000200:0.0:1788573179.202446:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573179.202453:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573179.202457:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573179.202463:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202469:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202474:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202479:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573179.202485:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573179.202489:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573179.202496:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573179.202499:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573179.202503:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573179.202507:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573179.202516:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202523:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202525:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202529:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573179.202537:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573179.202547:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202551:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573179.202558:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573179.202565:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573179.202569:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202578:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573179.202589:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573179.202596:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202601:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573179.202610:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573179.202617:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573179.202625:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573179.202629:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202632:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573179.202634:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573179.202672:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573179.202681:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573179.202689:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.202702:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810076032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573179.202726:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573179.202729:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.202737:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810076032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573179.202743:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573179.202746:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573179.202752:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573179.202755:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573179.202758:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573179.202761:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573179.202765:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13645 00000400:00000200:0.0:1788573179.202775:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573179.202787:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573179.202791:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573179.202794:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202796:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202797:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202804:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573179.202807:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573179.202808:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573179.202810:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573179.202813:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573179.202819:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573179.202821:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573179.202825:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202837:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573179.202841:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202850:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573179.202856:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573179.202861:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573179.202866:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573179.202875:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573179.202879:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573179.202886:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202892:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573179.202894:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573179.202906:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202908:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573179.202911:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573179.202919:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573179.202921:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573179.202922:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.202923:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573179.202924:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573179.202926:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573179.202931:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573179.202937:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.202948:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810076160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573179.202997:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573179.203002:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.203014:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810076160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573179.203027:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573179.203035:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13655) 00000020:00000001:0.0:1788573180.097843:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573180.097849:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573180.097853:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573183.234003:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573183.873777:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308322847, at 4308322880 00000100:00000010:0.0:1788573183.873801:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000020:00000001:0.0:1788573185.217800:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573185.217804:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573185.217808:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573185.346109:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573185.346114:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573185.346119:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573185.346129:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346150:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.346155:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.346160:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573185.346164:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573185.346168:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.346173:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810077056:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573185.346181:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573185.346186:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573185.346191:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.346198:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573185.346206:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573185.346216:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.346227:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573185.346232:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573185.346244:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573185.346253:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077056, portal 25 00000100:00000001:0.0:1788573185.346258:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573185.346261:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573185.346273:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346291:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573185.346299:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573185.346304:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573185.346310:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810077056, offset 0 00000400:00000200:0.0:1788573185.346322:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573185.346336:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.346346:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573185.346355:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573185.346362:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573185.346369:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573185.346374:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573185.346381:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540721:540721:256:4294967295] 192.168.202.148@tcp LPNI seq info [540686:540686:8:4294967295] 00000400:00000200:0.0:1788573185.346389:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.346404:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573185.346413:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.346422:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573185.346434:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573185.346440:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788573185.346466:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.346470:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573185.346480:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573185.346488:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573185.346503:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573185.346506:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573185.346512:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573185.346519:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346536:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.346543:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346562:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.346575:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573185.346583:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.346593:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573185.346599:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308328158 00000100:00000010:0.0:1788573185.346605:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573185.346615:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573185.346624:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308328158 00000100:00000010:0.0:1788573185.346631:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116267e0. 00000100:00000001:0.0:1788573185.346668:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573185.346679:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573185.346686:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.346694:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573185.346706:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573185/real 1788573185] req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573185.346735:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573185.346739:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573185.346745:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573185.346753:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573185.346758:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573185.346766:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573185.346775:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573185.346804:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346833:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573185.346840:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.346846:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.346851:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573185.346864:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346888:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573185.346898:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.346929:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573185.346936:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.346938:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.346940:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.346944:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573185.346946:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573185.346950:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573185.346956:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573185.346984:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573185.346991:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573185.347001:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810077056:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573185.347009:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347012:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.347016:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810077056/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347024:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573185.347028:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573185.347041:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573185.347045:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573185.347047:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573185.347049:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573185.347052:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573185.347055:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573185.347061:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573185.347063:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573185.347065:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573185.347069:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573185.347071:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347073:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347082:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573185.347084:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.347087:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573185.347088:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573185.347090:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573185.347095:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347102:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.347104:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347106:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573185.347108:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573185.347111:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.347114:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077184:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573185.347118:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573185.347120:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573185.347122:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347124:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573185.347127:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573185.347129:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.347134:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573185.347138:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573185.347142:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573185.347146:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077184, portal 10 00000100:00000001:0.0:1788573185.347153:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573185.347155:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573185.347164:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347171:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573185.347174:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573185.347177:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573185.347180:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810077184, offset 0 00000400:00000200:0.0:1788573185.347184:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347188:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.347192:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573185.347196:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573185.347199:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573185.347202:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573185.347204:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347208:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540722:540722:256:4294967295] 192.168.202.148@tcp LPNI seq info [540687:540687:8:4294967295] 00000400:00000200:0.0:1788573185.347212:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.347219:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573185.347224:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.347228:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573185.347232:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573185.347235:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788573185.347248:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.347250:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573185.347253:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573185.347257:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573185.347261:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573185.347263:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573185.347266:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573185.347270:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347291:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.347296:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347303:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.347304:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573185.347309:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573185.347314:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573185.347318:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573185.347320:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308324377 00000100:00000001:0.0:1788573185.347324:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.347326:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308324377 00000100:00000001:0.0:1788573185.347329:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.347331:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573185.347334:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.347336:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573185.347340:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573185/real 1788573185] req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573185.347350:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573185.347351:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573185.347353:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573185.347356:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573185.347358:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573185.347360:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573185.347364:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573185.347373:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347381:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573185.347388:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347390:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.347392:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573185.347396:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347404:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573185.347408:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347416:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573185.347419:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347420:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.347422:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347424:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573185.347426:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573185.347429:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573185.347433:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573185.347436:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573185.347440:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573185.347448:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077184:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573185.347453:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573185.347456:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347463:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.347465:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347467:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573185.347469:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573185.347471:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.347481:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077312:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573185.347485:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573185.347487:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573185.347489:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347491:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573185.347494:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573185.347496:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.347500:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573185.347503:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573185.347507:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573185.347511:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077312, portal 10 00000100:00000001:0.0:1788573185.347513:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573185.347514:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573185.347521:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347528:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573185.347531:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573185.347533:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573185.347536:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810077312, offset 0 00000400:00000200:0.0:1788573185.347540:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347544:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.347548:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573185.347552:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573185.347554:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573185.347557:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573185.347559:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347562:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540723:540723:256:4294967295] 192.168.202.148@tcp LPNI seq info [540688:540688:8:4294967295] 00000400:00000200:0.0:1788573185.347567:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.347590:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573185.347596:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.347600:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573185.347611:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573185.347614:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788573185.347623:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.347625:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573185.347628:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573185.347631:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573185.347634:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573185.347652:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573185.347654:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573185.347658:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347689:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.347693:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347700:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.347702:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573185.347706:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573185.347710:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573185.347714:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573185.347716:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308324382 00000100:00000001:0.0:1788573185.347718:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.347721:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308324382 00000100:00000001:0.0:1788573185.347723:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.347726:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573185.347728:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.347730:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573185.347734:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573185/real 1788573185] req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573185.347743:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573185.347750:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573185.347752:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573185.347755:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573185.347757:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573185.347761:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573185.347765:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573185.347774:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347781:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573185.347782:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347784:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.347786:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573185.347790:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347798:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573185.347802:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347810:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573185.347813:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347814:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.347816:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.347818:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573185.347820:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573185.347823:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573185.347827:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573185.347829:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573185.347833:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573185.347859:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077312:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573185.347863:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573185.347867:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347874:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.347875:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347878:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573185.347880:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573185.347882:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.347885:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077440:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573185.347888:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573185.347890:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573185.347892:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.347894:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573185.347897:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573185.347899:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.347904:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573185.347907:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573185.347911:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573185.347915:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077440, portal 4 00000100:00000001:0.0:1788573185.347917:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573185.347918:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573185.347925:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.347932:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573185.347934:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573185.347937:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573185.347940:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810077440, offset 0 00000400:00000200:0.0:1788573185.347944:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347955:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.347959:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573185.347977:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573185.347980:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573185.347983:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573185.347985:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573185.347988:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540724:540724:256:4294967295] 192.168.202.148@tcp LPNI seq info [540689:540689:8:4294967295] 00000400:00000200:0.0:1788573185.347993:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.348000:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573185.348005:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.348010:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573185.348014:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573185.348018:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788573185.348029:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.348031:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573185.348035:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573185.348039:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573185.348042:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573185.348044:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573185.348047:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573185.348051:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348060:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348063:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348070:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.348072:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573185.348076:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573185.348079:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573185.348082:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573185.348091:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308324514 00000100:00000001:0.0:1788573185.348094:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.348097:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308324514 00000100:00000001:0.0:1788573185.348099:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.348102:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573185.348104:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.348106:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573185.348111:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573185/real 1788573185] req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573185.348121:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573185.348122:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573185.348124:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573185.348127:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573185.348129:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573185.348132:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573185.348136:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573185.348145:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348152:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573185.348153:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348155:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.348156:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573185.348160:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348168:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573185.348172:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348180:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573185.348189:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348190:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.348192:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348194:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573185.348196:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573185.348199:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573185.348203:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573185.348205:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573185.348210:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573185.348219:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077440:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573185.348223:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573185.348227:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348234:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.348236:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.348238:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573185.348240:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573185.348242:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.348246:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077568:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573185.348250:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573185.348251:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573185.348253:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573185.348255:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573185.348259:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573185.348261:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573185.348265:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573185.348268:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573185.348272:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573185.348275:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077568, portal 4 00000100:00000001:0.0:1788573185.348283:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573185.348285:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573185.348292:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348299:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573185.348302:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573185.348305:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573185.348308:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810077568, offset 0 00000400:00000200:0.0:1788573185.348312:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573185.348317:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.348321:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573185.348324:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573185.348326:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573185.348329:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573185.348332:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573185.348335:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540725:540725:256:4294967295] 192.168.202.148@tcp LPNI seq info [540690:540690:8:4294967295] 00000400:00000200:0.0:1788573185.348340:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573185.348347:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573185.348352:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.348356:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573185.348359:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573185.348363:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788573185.348374:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573185.348377:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573185.348380:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573185.348383:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573185.348386:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573185.348388:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573185.348391:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573185.348395:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348415:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348419:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348426:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.348428:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573185.348432:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573185.348436:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573185.348440:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573185.348442:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308324523 00000100:00000001:0.0:1788573185.348445:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.348447:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308324523 00000100:00000001:0.0:1788573185.348450:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308328158 00000100:00000001:0.0:1788573185.348452:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573185.348455:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.348456:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573185.348461:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573185/real 1788573185] req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573185.348471:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573185.348472:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573185.348474:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573185.348477:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573185.348479:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573185.348482:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573185.348485:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573185.348494:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348501:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573185.348508:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348511:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.348512:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573185.348517:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348525:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573185.348529:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348537:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573185.348539:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348541:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573185.348543:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348545:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573185.348547:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573185.348549:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573185.348552:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573185.348555:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573185.348559:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573185.348568:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077568:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573185.348572:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348575:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348578:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810077184/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348585:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573185.348587:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573185.348590:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573185.348593:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573185.348594:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573185.348604:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573185.348606:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573185.348609:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573185.348612:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573185.348614:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573185.348615:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573185.348619:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573185.348620:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348622:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348627:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810077312/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348654:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573185.348657:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573185.348659:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573185.348662:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573185.348663:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573185.348664:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573185.348667:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573185.348669:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573185.348671:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573185.348672:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573185.348674:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573185.348677:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573185.348678:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348680:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348685:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810077440/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348692:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573185.348694:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573185.348695:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573185.348697:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573185.348699:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573185.348700:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573185.348702:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573185.348704:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573185.348706:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573185.348713:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573185.348715:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573185.348719:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573185.348720:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348722:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573185.348726:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810077568/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573185.348733:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573185.348734:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573185.348736:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573185.348739:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573185.348741:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573185.348742:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573185.348744:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573185.348747:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36c00. 02000000:00000001:0.0:1788573185.348749:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573185.348750:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573185.348752:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573185.348754:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573185.348756:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348758:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.348762:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573185.348764:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573185.348766:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573185.348768:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573189.185709:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308328158, at 4308328192 00000100:00000010:0.0:1788573189.185728:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573190.337881:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573190.337887:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573190.337892:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573190.466073:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573190.466084:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573190.466089:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573190.466104:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.466130:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.466150:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.466159:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573190.466168:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573190.466179:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.466194:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810077696:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573190.466209:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573190.466222:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573190.466229:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.466239:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573190.466254:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573190.466261:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.466286:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573190.466292:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573190.466314:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573190.466324:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077696, portal 25 00000100:00000001:0.0:1788573190.466329:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573190.466334:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286335232 : -109714423216384 : ffff9c371ec70700) 00000100:00000040:0.0:1788573190.466352:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.466373:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573190.466388:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573190.466393:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573190.466403:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810077696, offset 0 00000400:00000200:0.0:1788573190.466411:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573190.466424:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.466439:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573190.466445:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573190.466455:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573190.466465:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573190.466473:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573190.466485:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540726:540726:256:4294967295] 192.168.202.148@tcp LPNI seq info [540691:540691:8:4294967295] 00000400:00000200:0.0:1788573190.466510:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.466534:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573190.466549:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.466556:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573190.466568:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573190.466576:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573190.466612:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.466624:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573190.466687:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573190.466698:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573190.466705:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573190.466710:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573190.466720:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573190.466735:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.466770:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.466781:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.466810:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.466813:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573190.466824:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.466837:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573190.466844:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308333279 00000100:00000010:0.0:1788573190.466855:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573190.466868:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573190.466885:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308333279 00000100:00000010:0.0:1788573190.466894:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626240. 00000100:00000001:0.0:1788573190.466900:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573190.466905:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573190.466914:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.466926:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573190.466935:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573190/real 1788573190] req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573190.467006:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573190.467010:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573190.467013:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573190.467020:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573190.467031:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573190.467044:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573190.467059:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573190.467097:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467137:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573190.467146:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467156:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467167:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573190.467191:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467238:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573190.467256:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467282:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573190.467286:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467295:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467297:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467300:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573190.467302:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573190.467308:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573190.467314:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573190.467325:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573190.467330:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573190.467344:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810077696:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573190.467349:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467351:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.467356:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70700 x1875440810077696/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467362:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573190.467366:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573190.467369:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573190.467372:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573190.467374:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573190.467375:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573190.467377:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573190.467380:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d000. 02000000:00000001:0.0:1788573190.467384:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573190.467386:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573190.467387:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70700. 00000100:00000001:0.0:1788573190.467390:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573190.467391:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467393:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467400:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573190.467402:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.467404:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573190.467405:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573190.467407:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573190.467410:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467415:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.467417:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467418:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573190.467420:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573190.467422:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.467424:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077824:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573190.467429:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573190.467431:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573190.467433:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467434:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573190.467436:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573190.467438:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.467442:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573190.467444:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573190.467447:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573190.467450:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077824, portal 10 00000100:00000001:0.0:1788573190.467451:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573190.467452:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286338816 : -109714423212800 : ffff9c371ec71500) 00000100:00000040:0.0:1788573190.467458:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467463:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573190.467465:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573190.467466:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573190.467468:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810077824, offset 0 00000400:00000200:0.0:1788573190.467471:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573190.467474:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.467477:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573190.467479:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573190.467481:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573190.467483:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573190.467485:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573190.467487:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540727:540727:256:4294967295] 192.168.202.148@tcp LPNI seq info [540692:540692:8:4294967295] 00000400:00000200:0.0:1788573190.467490:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.467495:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573190.467498:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.467500:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573190.467503:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573190.467511:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573190.467519:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.467520:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573190.467522:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573190.467525:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573190.467528:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573190.467529:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573190.467531:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573190.467534:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467540:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.467542:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467547:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.467548:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573190.467550:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573190.467554:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573190.467556:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573190.467558:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308329497 00000100:00000001:0.0:1788573190.467560:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.467567:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308329497 00000100:00000001:0.0:1788573190.467568:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.467570:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573190.467572:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.467573:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573190.467576:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573190/real 1788573190] req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573190.467582:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573190.467583:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573190.467587:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573190.467589:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573190.467590:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573190.467592:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573190.467595:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573190.467601:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467605:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573190.467606:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467607:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467608:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573190.467610:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467615:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573190.467618:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467622:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573190.467623:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467624:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467625:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467626:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573190.467627:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573190.467629:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573190.467631:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573190.467632:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573190.467653:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573190.467659:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077824:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573190.467666:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573190.467669:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467674:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.467675:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467677:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573190.467679:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573190.467680:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.467682:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077952:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573190.467684:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573190.467685:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573190.467686:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467688:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573190.467690:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573190.467692:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.467694:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573190.467696:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573190.467698:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573190.467701:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810077952, portal 10 00000100:00000001:0.0:1788573190.467702:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573190.467703:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573190.467708:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467712:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573190.467714:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573190.467715:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573190.467717:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810077952, offset 0 00000400:00000200:0.0:1788573190.467719:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573190.467722:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.467725:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573190.467727:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573190.467735:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573190.467738:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573190.467740:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573190.467744:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540728:540728:256:4294967295] 192.168.202.148@tcp LPNI seq info [540693:540693:8:4294967295] 00000400:00000200:0.0:1788573190.467748:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.467756:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573190.467761:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.467765:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573190.467769:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573190.467772:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573190.467797:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.467798:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573190.467800:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573190.467803:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573190.467806:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573190.467807:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573190.467809:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573190.467812:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467818:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.467820:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467827:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.467829:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573190.467832:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573190.467836:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573190.467840:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573190.467842:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308329502 00000100:00000001:0.0:1788573190.467845:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.467847:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308329502 00000100:00000001:0.0:1788573190.467856:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.467859:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573190.467861:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.467863:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573190.467868:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573190/real 1788573190] req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573190.467877:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573190.467878:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573190.467879:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573190.467881:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573190.467883:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573190.467884:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573190.467887:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573190.467892:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467897:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573190.467898:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467899:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467900:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573190.467903:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467908:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573190.467910:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467914:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573190.467916:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467917:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.467918:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.467922:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573190.467924:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573190.467926:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573190.467929:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573190.467930:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573190.467933:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573190.467939:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810077952:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573190.467941:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573190.467944:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467948:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.467949:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467951:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573190.467952:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573190.467954:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.467956:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078080:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573190.467958:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573190.467959:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573190.467961:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.467962:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573190.467964:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573190.467965:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.467969:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573190.467970:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573190.467973:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573190.467975:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078080, portal 4 00000100:00000001:0.0:1788573190.467977:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573190.467978:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286336128 : -109714423215488 : ffff9c371ec70a80) 00000100:00000040:0.0:1788573190.467983:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.467990:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573190.467992:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573190.467994:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573190.467996:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810078080, offset 0 00000400:00000200:0.0:1788573190.467998:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573190.468001:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.468004:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573190.468006:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573190.468008:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573190.468010:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573190.468011:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573190.468014:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540729:540729:256:4294967295] 192.168.202.148@tcp LPNI seq info [540694:540694:8:4294967295] 00000400:00000200:0.0:1788573190.468016:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.468021:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573190.468024:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.468026:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573190.468029:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573190.468031:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573190.468038:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.468040:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573190.468042:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573190.468044:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573190.468046:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573190.468048:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573190.468049:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573190.468052:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468057:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468059:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468073:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.468074:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573190.468077:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573190.468079:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573190.468082:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573190.468083:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308329634 00000100:00000001:0.0:1788573190.468085:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.468086:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308329634 00000100:00000001:0.0:1788573190.468088:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.468089:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573190.468090:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.468092:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573190.468094:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573190/real 1788573190] req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573190.468099:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573190.468100:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573190.468101:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573190.468102:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573190.468104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573190.468105:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573190.468107:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573190.468112:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468116:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573190.468117:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468118:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.468119:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573190.468121:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468129:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573190.468132:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468136:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573190.468137:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468138:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.468139:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468140:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573190.468142:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573190.468144:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573190.468146:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573190.468147:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573190.468150:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573190.468155:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078080:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573190.468157:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573190.468160:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468164:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.468165:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.468166:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573190.468168:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573190.468169:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.468170:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078208:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573190.468172:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573190.468174:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573190.468177:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573190.468179:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573190.468181:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573190.468183:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573190.468186:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573190.468189:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573190.468193:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573190.468196:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078208, portal 4 00000100:00000001:0.0:1788573190.468198:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573190.468200:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345984 : -109714423205632 : ffff9c371ec73100) 00000100:00000040:0.0:1788573190.468207:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468214:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573190.468217:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573190.468219:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573190.468222:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810078208, offset 0 00000400:00000200:0.0:1788573190.468226:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573190.468230:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.468234:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573190.468237:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573190.468240:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573190.468242:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573190.468244:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573190.468248:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540730:540730:256:4294967295] 192.168.202.148@tcp LPNI seq info [540695:540695:8:4294967295] 00000400:00000200:0.0:1788573190.468252:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573190.468258:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573190.468263:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.468266:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573190.468270:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573190.468273:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788573190.468286:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573190.468290:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573190.468292:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573190.468295:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573190.468298:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573190.468299:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573190.468301:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573190.468304:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468309:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468311:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468316:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.468317:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573190.468319:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573190.468322:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573190.468324:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573190.468325:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308329643 00000100:00000001:0.0:1788573190.468327:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.468329:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308329643 00000100:00000001:0.0:1788573190.468330:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308333279 00000100:00000001:0.0:1788573190.468332:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573190.468333:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.468334:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573190.468336:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573190/real 1788573190] req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573190.468342:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573190.468342:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573190.468344:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573190.468346:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573190.468347:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573190.468352:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573190.468355:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573190.468360:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468364:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573190.468365:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468366:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.468367:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573190.468369:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468373:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573190.468376:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468381:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573190.468382:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468383:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573190.468384:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468385:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573190.468386:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573190.468387:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573190.468390:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573190.468391:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573190.468393:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573190.468398:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078208:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573190.468400:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468402:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468407:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71500 x1875440810077824/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468412:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573190.468413:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573190.468414:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573190.468416:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573190.468417:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573190.468418:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573190.468420:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573190.468421:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788573190.468423:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573190.468424:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573190.468425:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71500. 00000100:00000001:0.0:1788573190.468427:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573190.468428:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468429:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468431:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810077952/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468435:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573190.468436:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573190.468437:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573190.468439:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573190.468439:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573190.468440:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573190.468441:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573190.468442:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f400. 02000000:00000001:0.0:1788573190.468443:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573190.468444:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573190.468445:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573190.468446:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573190.468447:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468448:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468450:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70a80 x1875440810078080/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468453:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573190.468458:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573190.468459:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573190.468461:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573190.468462:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573190.468463:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573190.468465:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573190.468465:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2ec00. 02000000:00000001:0.0:1788573190.468467:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573190.468467:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573190.468468:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70a80. 00000100:00000001:0.0:1788573190.468470:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573190.468470:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468471:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573190.468473:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73100 x1875440810078208/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573190.468477:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573190.468478:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573190.468479:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573190.468480:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573190.468481:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573190.468482:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573190.468483:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573190.468483:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2dc00. 02000000:00000001:0.0:1788573190.468484:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573190.468485:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573190.468486:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73100. 00000100:00000001:0.0:1788573190.468487:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573190.468488:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468489:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.468492:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573190.468493:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573190.468494:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573190.468495:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.305721:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308333279, at 4308333312 00000100:00000010:0.0:1788573194.305739:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573195.457898:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573195.457922:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573195.457926:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573195.586015:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573195.586021:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573195.586027:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573195.586037:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586053:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.586056:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.586063:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573195.586066:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573195.586073:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.586079:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810078336:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573195.586086:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573195.586092:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573195.586097:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.586100:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573195.586106:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573195.586110:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.586122:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573195.586130:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573195.586144:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573195.586155:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078336, portal 25 00000100:00000001:0.0:1788573195.586162:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573195.586167:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788573195.586181:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586198:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573195.586205:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573195.586210:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573195.586217:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810078336, offset 0 00000400:00000200:0.0:1788573195.586224:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573195.586236:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.586247:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573195.586263:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573195.586271:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573195.586279:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573195.586284:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573195.586292:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540731:540731:256:4294967295] 192.168.202.148@tcp LPNI seq info [540696:540696:8:4294967295] 00000400:00000200:0.0:1788573195.586301:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.586316:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573195.586329:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.586337:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573195.586348:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573195.586355:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573195.586383:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.586387:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573195.586396:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573195.586406:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573195.586413:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573195.586417:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573195.586425:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573195.586435:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586466:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.586476:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586500:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.586504:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573195.586514:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.586523:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573195.586528:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308338398 00000100:00000010:0.0:1788573195.586533:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573195.586543:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573195.586564:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308338398 00000100:00000010:0.0:1788573195.586571:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626ea0. 00000100:00000001:0.0:1788573195.586579:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573195.586586:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573195.586592:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.586598:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573195.586623:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573195/real 1788573195] req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573195.586688:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573195.586693:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573195.586698:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573195.586706:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573195.586712:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573195.586721:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573195.586731:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573195.586751:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586767:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573195.586770:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586773:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.586776:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573195.586781:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586793:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573195.586804:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586818:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573195.586823:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586827:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.586836:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586840:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573195.586841:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573195.586844:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573195.586849:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573195.586860:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573195.586865:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573195.586872:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810078336:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573195.586882:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586884:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.586887:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a300 x1875440810078336/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586893:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573195.586896:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573195.586898:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573195.586901:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573195.586903:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573195.586905:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573195.586907:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573195.586909:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36c00. 02000000:00000001:0.0:1788573195.586913:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573195.586915:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573195.586916:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a300. 00000100:00000001:0.0:1788573195.586919:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573195.586920:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586922:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.586929:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573195.586930:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.586932:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573195.586933:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573195.586935:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573195.586937:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.586946:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.586947:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.586949:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573195.586951:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573195.586952:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.586954:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078464:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573195.586957:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573195.586958:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573195.586960:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.586961:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573195.586963:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573195.586964:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.586969:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573195.586972:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573195.586977:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573195.586980:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078464, portal 10 00000100:00000001:0.0:1788573195.586982:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573195.586984:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287363584 : -109714422188032 : ffff9c371ed6b800) 00000100:00000040:0.0:1788573195.586992:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587000:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573195.587002:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573195.587005:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573195.587008:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810078464, offset 0 00000400:00000200:0.0:1788573195.587012:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587017:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587021:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573195.587023:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573195.587025:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573195.587027:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573195.587029:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587035:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540732:540732:256:4294967295] 192.168.202.148@tcp LPNI seq info [540697:540697:8:4294967295] 00000400:00000200:0.0:1788573195.587039:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587044:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573195.587048:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587051:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573195.587054:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573195.587056:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573195.587064:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587065:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573195.587068:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573195.587070:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573195.587073:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573195.587074:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573195.587076:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573195.587079:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587085:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587087:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587092:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587093:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573195.587095:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573195.587099:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573195.587101:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573195.587102:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308334617 00000100:00000001:0.0:1788573195.587105:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587106:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308334617 00000100:00000001:0.0:1788573195.587108:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587110:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573195.587116:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587118:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573195.587121:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573195/real 1788573195] req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573195.587127:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573195.587128:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573195.587130:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573195.587131:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573195.587132:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573195.587134:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573195.587136:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573195.587142:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587145:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573195.587146:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587147:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587148:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573195.587150:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587154:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573195.587157:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587162:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573195.587164:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587165:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587166:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587167:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573195.587168:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573195.587170:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573195.587175:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573195.587177:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573195.587180:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573195.587186:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078464:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573195.587189:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573195.587191:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587195:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.587196:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587197:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573195.587198:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573195.587200:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.587201:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078592:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573195.587203:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573195.587204:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573195.587205:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587207:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587209:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573195.587211:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.587213:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573195.587215:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573195.587217:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573195.587219:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078592, portal 10 00000100:00000001:0.0:1788573195.587221:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573195.587222:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360000 : -109714422191616 : ffff9c371ed6aa00) 00000100:00000040:0.0:1788573195.587226:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587231:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573195.587235:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573195.587237:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573195.587239:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810078592, offset 0 00000400:00000200:0.0:1788573195.587242:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587244:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587247:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573195.587249:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573195.587251:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573195.587253:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573195.587255:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587257:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540733:540733:256:4294967295] 192.168.202.148@tcp LPNI seq info [540698:540698:8:4294967295] 00000400:00000200:0.0:1788573195.587260:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587264:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573195.587267:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587269:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573195.587271:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573195.587273:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573195.587280:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587282:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573195.587284:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573195.587286:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573195.587288:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573195.587290:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573195.587291:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573195.587294:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587299:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587301:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587306:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587310:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573195.587313:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573195.587315:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573195.587318:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573195.587319:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308334621 00000100:00000001:0.0:1788573195.587321:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587323:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308334621 00000100:00000001:0.0:1788573195.587324:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587326:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573195.587327:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587328:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573195.587330:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573195/real 1788573195] req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573195.587335:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573195.587336:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573195.587338:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573195.587339:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573195.587340:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573195.587342:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573195.587344:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573195.587349:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587354:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573195.587355:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587356:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587357:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573195.587359:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587363:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573195.587368:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587373:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573195.587375:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587376:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587377:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587378:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573195.587379:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573195.587381:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573195.587382:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573195.587384:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573195.587386:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573195.587392:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078592:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573195.587394:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573195.587396:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587400:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.587401:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587402:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573195.587404:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573195.587405:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.587406:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078720:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573195.587408:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573195.587409:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573195.587410:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587411:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587413:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573195.587415:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.587420:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573195.587422:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573195.587425:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573195.587427:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078720, portal 4 00000100:00000001:0.0:1788573195.587428:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573195.587429:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287362688 : -109714422188928 : ffff9c371ed6b480) 00000100:00000040:0.0:1788573195.587434:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587439:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573195.587440:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573195.587442:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573195.587443:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810078720, offset 0 00000400:00000200:0.0:1788573195.587446:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587448:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587450:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573195.587452:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573195.587453:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573195.587455:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573195.587457:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587459:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540734:540734:256:4294967295] 192.168.202.148@tcp LPNI seq info [540699:540699:8:4294967295] 00000400:00000200:0.0:1788573195.587462:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587466:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573195.587469:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587470:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573195.587473:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573195.587474:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573195.587481:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587483:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573195.587485:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573195.587487:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573195.587489:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573195.587493:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573195.587495:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573195.587497:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587503:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587505:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587509:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587511:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573195.587513:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573195.587515:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573195.587517:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573195.587518:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308334753 00000100:00000001:0.0:1788573195.587520:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587522:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308334753 00000100:00000001:0.0:1788573195.587523:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587524:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573195.587526:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587527:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573195.587529:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573195/real 1788573195] req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573195.587533:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573195.587534:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573195.587535:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573195.587537:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573195.587538:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573195.587539:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573195.587541:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573195.587549:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587562:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573195.587564:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587566:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587568:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573195.587573:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587580:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573195.587585:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587593:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573195.587595:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587597:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587599:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587600:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573195.587615:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573195.587618:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573195.587621:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573195.587622:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573195.587626:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573195.587631:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078720:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573195.587633:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573195.587650:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587655:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.587656:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587663:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573195.587664:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573195.587665:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.587668:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078848:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573195.587670:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573195.587672:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573195.587673:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573195.587674:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587676:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573195.587678:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573195.587681:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573195.587682:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573195.587685:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573195.587687:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078848, portal 4 00000100:00000001:0.0:1788573195.587689:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573195.587690:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573195.587694:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587699:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573195.587700:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573195.587702:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573195.587703:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810078848, offset 0 00000400:00000200:0.0:1788573195.587706:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587708:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587711:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573195.587713:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573195.587714:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573195.587716:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573195.587718:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573195.587720:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540735:540735:256:4294967295] 192.168.202.148@tcp LPNI seq info [540700:540700:8:4294967295] 00000400:00000200:0.0:1788573195.587723:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573195.587728:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573195.587735:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587737:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573195.587740:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573195.587742:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573195.587750:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573195.587752:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573195.587754:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573195.587756:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573195.587758:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573195.587759:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573195.587761:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573195.587764:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587769:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587771:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587776:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587777:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573195.587779:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573195.587782:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573195.587784:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573195.587786:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308334763 00000100:00000001:0.0:1788573195.587788:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587789:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308334763 00000100:00000001:0.0:1788573195.587791:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308338398 00000100:00000001:0.0:1788573195.587792:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573195.587793:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587794:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573195.587796:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573195/real 1788573195] req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573195.587806:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573195.587806:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573195.587808:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573195.587809:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573195.587810:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573195.587812:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573195.587814:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573195.587820:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587824:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573195.587825:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587826:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587827:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573195.587829:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587833:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573195.587835:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587840:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573195.587842:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587843:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573195.587844:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587845:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573195.587846:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573195.587848:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573195.587849:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573195.587851:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573195.587856:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573195.587862:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810078848:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573195.587864:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587865:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587867:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b800 x1875440810078464/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587871:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573195.587872:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587874:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573195.587875:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573195.587876:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573195.587877:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573195.587879:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573195.587880:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573195.587882:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573195.587883:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573195.587884:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b800. 00000100:00000001:0.0:1788573195.587886:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573195.587887:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587888:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587890:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6aa00 x1875440810078592/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587894:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573195.587895:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587896:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573195.587897:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573195.587898:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573195.587899:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573195.587900:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573195.587901:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573195.587902:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573195.587905:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573195.587907:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6aa00. 00000100:00000001:0.0:1788573195.587908:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573195.587909:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587911:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587913:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b480 x1875440810078720/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587917:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573195.587918:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587919:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573195.587920:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573195.587921:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573195.587922:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573195.587923:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573195.587924:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573195.587925:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573195.587926:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573195.587926:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b480. 00000100:00000001:0.0:1788573195.587928:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573195.587928:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587929:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573195.587931:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810078848/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573195.587935:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573195.587935:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573195.587936:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573195.587938:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573195.587938:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573195.587939:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573195.587940:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573195.587941:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573195.587942:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573195.587943:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573195.587943:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573195.587945:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573195.587945:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587949:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.587952:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573195.587953:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573195.587955:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573195.587956:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573199.425760:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308338398, at 4308338432 00000100:00000010:0.0:1788573199.425777:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573200.577926:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573200.577932:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573200.577937:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573200.706095:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573200.706106:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573200.706111:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573200.706133:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.706168:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.706177:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.706185:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573200.706189:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573200.706197:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.706204:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810078976:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573200.706213:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573200.706220:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573200.706226:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.706230:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573200.706241:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.706246:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.706263:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573200.706267:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573200.706274:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573200.706290:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810078976, portal 25 00000100:00000001:0.0:1788573200.706301:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573200.706304:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788573200.706313:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.706344:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573200.706349:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573200.706351:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573200.706354:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810078976, offset 0 00000400:00000200:0.0:1788573200.706370:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573200.706391:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.706405:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573200.706415:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573200.706430:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573200.706438:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573200.706443:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573200.706465:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540736:540736:256:4294967295] 192.168.202.148@tcp LPNI seq info [540701:540701:8:4294967295] 00000400:00000200:0.0:1788573200.706480:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.706495:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573200.706506:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.706518:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573200.706531:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573200.706539:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573200.706569:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.706572:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573200.706579:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573200.706586:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573200.706591:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573200.706600:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573200.706611:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573200.706620:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.706679:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.706684:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.706718:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.706721:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573200.706731:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.706741:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573200.706745:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308343519 00000100:00000010:0.0:1788573200.706751:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573200.706759:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573200.706767:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308343519 00000100:00000010:0.0:1788573200.706774:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626f60. 00000100:00000001:0.0:1788573200.706780:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573200.706787:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573200.706792:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.706803:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573200.706817:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573200/real 1788573200] req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573200.706843:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573200.706847:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573200.706853:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573200.706865:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573200.706871:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573200.706879:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573200.706888:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573200.706919:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.706938:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573200.706943:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.706946:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.706948:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573200.706959:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707003:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573200.707018:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707041:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573200.707047:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707053:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.707057:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707063:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573200.707067:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573200.707074:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573200.707083:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573200.707090:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573200.707103:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573200.707115:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810078976:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573200.707123:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707129:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.707137:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440810078976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707163:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573200.707170:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573200.707176:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.707183:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573200.707187:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573200.707190:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573200.707200:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573200.707203:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233ec00. 02000000:00000001:0.0:1788573200.707207:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573200.707220:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573200.707222:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788573200.707225:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573200.707226:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707228:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707234:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573200.707235:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.707237:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573200.707238:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573200.707239:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573200.707244:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707252:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.707257:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.707259:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573200.707263:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573200.707267:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.707274:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079104:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573200.707289:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573200.707294:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573200.707297:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.707304:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573200.707310:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.707315:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.707332:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573200.707341:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573200.707354:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573200.707367:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810079104, portal 10 00000100:00000001:0.0:1788573200.707378:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573200.707384:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788573200.707402:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707435:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573200.707445:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573200.707468:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573200.707477:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810079104, offset 0 00000400:00000200:0.0:1788573200.707494:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573200.707506:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.707514:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573200.707523:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573200.707530:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573200.707542:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573200.707545:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573200.707556:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540737:540737:256:4294967295] 192.168.202.148@tcp LPNI seq info [540702:540702:8:4294967295] 00000400:00000200:0.0:1788573200.707575:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.707605:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573200.707629:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.707668:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573200.707677:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573200.707685:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573200.707726:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.707729:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573200.707737:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573200.707750:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573200.707759:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573200.707765:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573200.707771:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573200.707782:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707797:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.707801:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707809:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.707811:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573200.707822:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573200.707826:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573200.707830:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573200.707832:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308339738 00000100:00000001:0.0:1788573200.707835:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.707838:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308339738 00000100:00000001:0.0:1788573200.707840:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.707842:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573200.707845:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.707847:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573200.707851:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573200/real 1788573200] req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573200.707861:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573200.707862:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573200.707864:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573200.707867:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573200.707869:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573200.707871:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573200.707875:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573200.707884:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707892:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573200.707893:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707895:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.707897:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573200.707900:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707907:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573200.707911:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707923:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573200.707926:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707927:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.707929:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.707930:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573200.707932:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573200.707935:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573200.707938:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573200.707940:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573200.707944:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573200.707951:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079104:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573200.707955:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573200.707959:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.707966:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.707968:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.707970:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573200.707973:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573200.707975:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.707977:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079232:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573200.707980:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573200.707983:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573200.707984:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.707986:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573200.707990:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573200.707991:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.708002:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573200.708004:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573200.708008:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573200.708011:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810079232, portal 10 00000100:00000001:0.0:1788573200.708013:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573200.708014:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000040:0.0:1788573200.708021:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708027:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573200.708030:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573200.708032:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573200.708034:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810079232, offset 0 00000400:00000200:0.0:1788573200.708039:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708043:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708047:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573200.708050:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573200.708053:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573200.708056:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573200.708059:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708062:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540738:540738:256:4294967295] 192.168.202.148@tcp LPNI seq info [540703:540703:8:4294967295] 00000400:00000200:0.0:1788573200.708067:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708073:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573200.708079:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708082:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573200.708086:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573200.708089:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573200.708100:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708102:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573200.708105:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573200.708109:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573200.708112:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573200.708119:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573200.708121:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573200.708125:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708135:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708138:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708145:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708147:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573200.708151:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573200.708155:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573200.708158:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573200.708161:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308339742 00000100:00000001:0.0:1788573200.708163:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708166:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308339742 00000100:00000001:0.0:1788573200.708168:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708170:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573200.708172:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708175:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573200.708179:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573200/real 1788573200] req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573200.708186:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573200.708187:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573200.708189:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573200.708190:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573200.708191:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573200.708193:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573200.708195:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573200.708201:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708209:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573200.708211:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708212:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708213:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573200.708216:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708220:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573200.708223:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708227:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573200.708229:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708230:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708230:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708232:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573200.708233:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573200.708234:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573200.708236:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573200.708237:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573200.708240:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573200.708244:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079232:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573200.708247:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708248:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708250:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440810079104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708255:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573200.708258:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708260:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.708262:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573200.708263:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573200.708264:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573200.708265:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573200.708267:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788573200.708269:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573200.708270:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573200.708271:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788573200.708273:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573200.708274:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708275:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708277:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440810079232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708281:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573200.708282:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708283:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573200.708284:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573200.708285:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573200.708286:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=3 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573200.708287:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573200.708288:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233cc00. 02000000:00000001:0.0:1788573200.708289:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573200.708290:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573200.708291:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788573200.708292:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573200.708293:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708294:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708297:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573200.708297:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708299:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573200.708300:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573200.708301:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573200.708304:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708308:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.708312:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.708314:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573200.708315:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573200.708316:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.708319:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079360:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573200.708321:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573200.708323:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573200.708324:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.708325:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708327:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573200.708328:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.708331:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573200.708333:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573200.708335:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573200.708337:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810079360, portal 4 00000100:00000001:0.0:1788573200.708339:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573200.708340:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788573200.708344:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708348:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573200.708350:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573200.708351:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573200.708352:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810079360, offset 0 00000400:00000200:0.0:1788573200.708355:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708357:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708360:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573200.708362:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573200.708363:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573200.708365:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573200.708367:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708369:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540739:540739:256:4294967295] 192.168.202.148@tcp LPNI seq info [540704:540704:8:4294967295] 00000400:00000200:0.0:1788573200.708372:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708376:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573200.708382:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708385:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573200.708387:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573200.708389:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573200.708397:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708398:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573200.708400:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573200.708402:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573200.708404:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573200.708405:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573200.708407:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573200.708409:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708415:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708416:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708421:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708422:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573200.708424:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573200.708427:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573200.708429:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573200.708430:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308339874 00000100:00000001:0.0:1788573200.708432:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708433:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308339874 00000100:00000001:0.0:1788573200.708434:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708436:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573200.708437:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708438:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573200.708440:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573200/real 1788573200] req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573200.708449:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573200.708463:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573200.708464:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573200.708466:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573200.708467:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573200.708469:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573200.708472:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573200.708478:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708482:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573200.708483:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708484:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708485:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573200.708487:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708491:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573200.708493:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708498:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573200.708500:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708501:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708501:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708502:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573200.708504:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573200.708506:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573200.708508:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573200.708509:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573200.708517:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573200.708522:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079360:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573200.708525:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573200.708527:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708531:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.708532:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.708534:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573200.708535:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573200.708536:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.708538:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079488:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573200.708540:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573200.708542:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573200.708543:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573200.708544:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708546:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.708547:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573200.708550:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573200.708551:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573200.708553:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573200.708556:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810079488, portal 4 00000100:00000001:0.0:1788573200.708557:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573200.708558:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788573200.708562:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708566:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573200.708568:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573200.708569:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573200.708570:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810079488, offset 0 00000400:00000200:0.0:1788573200.708576:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708579:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708581:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573200.708583:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573200.708585:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573200.708586:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573200.708588:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573200.708590:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540740:540740:256:4294967295] 192.168.202.148@tcp LPNI seq info [540705:540705:8:4294967295] 00000400:00000200:0.0:1788573200.708593:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573200.708597:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573200.708599:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708601:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573200.708604:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573200.708606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573200.708613:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573200.708614:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573200.708616:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573200.708618:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573200.708620:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573200.708621:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573200.708622:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573200.708625:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708630:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708632:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573225 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708655:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708656:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573200.708658:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573200.708661:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573200.708663:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573200.708668:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308339884 00000100:00000001:0.0:1788573200.708670:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708672:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308339884 00000100:00000001:0.0:1788573200.708673:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308343519 00000100:00000001:0.0:1788573200.708675:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573200.708676:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708677:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573200.708680:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573200/real 1788573200] req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573200.708685:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573200.708686:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573200.708688:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573200.708690:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573200.708692:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573200.708694:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573200.708698:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573200.708707:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708714:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573200.708716:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708718:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708719:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573200.708723:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708730:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573200.708734:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708741:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573200.708750:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708752:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573200.708753:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708755:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573200.708757:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573200.708760:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573200.708764:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573200.708766:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573200.708770:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573200.708779:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810079488:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573200.708782:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708784:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708787:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810079360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708795:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573200.708796:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708799:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573200.708801:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573200.708803:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573200.708805:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=5 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573200.708806:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573200.708808:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e800. 02000000:00000001:0.0:1788573200.708811:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573200.708812:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573200.708814:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788573200.708816:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573200.708818:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708820:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573200.708823:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810079488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573225 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573200.708841:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573200.708843:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573200.708845:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573200.708847:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573200.708849:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573200.708850:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573200.708852:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573200.708853:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788573200.708856:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573200.708857:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573200.708859:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788573200.708861:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573200.708862:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708865:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.708868:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573200.708870:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.708872:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573200.708873:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.282705:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573201.282714:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.282717:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573201.282731:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.282744:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.282747:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573201.282751:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573201.282755:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573201.282758:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573201.282763:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810079744:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573201.282768:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573201.282773:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573201.282776:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573201.282778:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573201.282788:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573201.282805:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573201.282814:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573201.282819:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573201.282827:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573201.282836:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810079744, portal 25 00000100:00000001:0.0:1788573201.282839:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573201.282841:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291926656 : -109714417624960 : ffff9c371f1c5880) 00000100:00000040:0.0:1788573201.282848:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.282857:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573201.282861:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573201.282864:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573201.282867:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810079744, offset 0 00000400:00000200:0.0:1788573201.282874:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573201.282883:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573201.282892:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573201.282898:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573201.282902:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573201.282906:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573201.282909:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573201.282913:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540741:540741:256:4294967295] 192.168.202.148@tcp LPNI seq info [540706:540706:8:4294967295] 00000400:00000200:0.0:1788573201.282919:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573201.282942:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573201.282949:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573201.282956:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573201.282963:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573201.282967:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 31 secs later) 00000800:00000100:0.0:1788573201.282983:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573201.282985:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573201.282991:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573201.282996:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573201.283009:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573201.283012:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573201.283016:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573201.283032:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283044:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573201.283048:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283056:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.283059:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573201.283065:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573201.283070:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573201.283073:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308344095 00000100:00000001:0.0:1788573201.283083:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308343519 00000100:00000001:0.0:1788573201.283089:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308344095 00000100:00000001:0.0:1788573201.283093:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308343519 00000100:00000001:0.0:1788573201.283097:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573201.283100:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.283102:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573201.283107:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573201/real 1788573201] req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573201.283118:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573201.283120:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573201.283123:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573201.283126:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573201.283128:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573201.283131:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573201.283134:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573201.283144:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283167:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573201.283170:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283173:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573201.283175:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573201.283180:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283188:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573201.283194:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283201:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573201.283205:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283207:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573201.283208:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283211:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573201.283214:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573201.283217:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573201.283223:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573201.283242:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573201.283248:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573201.283258:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810079744:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573201.283273:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283277:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573201.283282:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440810079744/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.283289:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573201.283292:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573201.283302:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573201.283305:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573201.283308:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573201.283311:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573201.283313:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573201.283316:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788573201.283320:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573201.283322:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573201.283324:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788573201.283328:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573201.283329:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283332:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.283339:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573201.283342:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.283344:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573201.283346:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.602788:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573201.602810:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.602819:0:2403:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573201.602837:0:2403:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7b80 x1875440810080000/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.602923:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.602929:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788573201.602945:0:2403:0:(client.c:1853:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9c371f1c7b80 x1875440810080000/t0(0) o103->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573201.602996:0:2403:0:(client.c:1858:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603003:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603007:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.603009:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603012:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603014:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.603016:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603019:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603022:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603030:0:2403:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573201.603032:0:2403:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603053:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573201.603054:0:2403:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.603055:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603056:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.603057:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603059:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603060:0:2403:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.603061:0:2403:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603062:0:2403:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603065:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603074:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573201.603075:0:2404:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.603076:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603077:0:2404:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.603078:0:2404:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.603080:0:2404:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.603081:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573203.649836:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573203.649841:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573203.649845:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573205.121787:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308344095, at 4308344128 00000100:00000010:0.0:1788573205.121803:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573206.337921:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573206.337927:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573206.337930:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573206.337940:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.337956:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.337960:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.337965:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573206.337970:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573206.337974:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.337981:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810080128:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573206.337989:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573206.337994:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573206.338004:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.338009:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573206.338014:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573206.338017:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.338025:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573206.338030:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573206.338038:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573206.338046:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080128, portal 25 00000100:00000001:0.0:1788573206.338051:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573206.338054:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788573206.338063:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338074:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573206.338079:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573206.338082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573206.338087:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810080128, offset 0 00000400:00000200:0.0:1788573206.338098:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573206.338109:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.338120:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573206.338127:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573206.338133:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573206.338140:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573206.338145:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573206.338148:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540742:540742:256:4294967295] 192.168.202.148@tcp LPNI seq info [540707:540707:8:4294967295] 00000400:00000200:0.0:1788573206.338151:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.338158:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573206.338166:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.338170:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573206.338180:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573206.338188:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 25 secs later) 00000800:00000100:0.0:1788573206.338221:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.338229:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573206.338238:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573206.338247:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573206.338254:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573206.338258:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573206.338265:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573206.338271:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338288:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.338292:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338304:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.338306:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573206.338314:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.338322:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573206.338327:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308349150 00000100:00000010:0.0:1788573206.338335:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573206.338368:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573206.338377:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308349150 00000100:00000010:0.0:1788573206.338383:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626120. 00000100:00000001:0.0:1788573206.338389:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573206.338395:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573206.338400:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.338406:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573206.338413:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573206/real 1788573206] req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573206.338422:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573206.338426:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573206.338429:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573206.338433:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573206.338435:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573206.338439:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573206.338452:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573206.338471:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338480:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573206.338483:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338486:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.338489:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573206.338496:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338508:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573206.338516:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338528:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573206.338532:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338534:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.338536:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338539:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573206.338542:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573206.338548:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573206.338556:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573206.338572:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573206.338579:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573206.338590:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810080128:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573206.338597:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338602:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.338605:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440810080128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338649:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573206.338714:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573206.338719:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573206.338727:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573206.338731:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573206.338735:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573206.338743:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573206.338747:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788573206.338755:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573206.338761:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573206.338766:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788573206.338773:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573206.338775:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338782:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.338792:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573206.338798:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.338814:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573206.338818:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573206.338821:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573206.338833:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338863:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.338867:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.338872:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573206.338875:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573206.338878:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.338883:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080256:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573206.338888:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573206.338890:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573206.338892:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.338894:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573206.338898:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573206.338900:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.338905:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573206.338915:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573206.338919:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573206.338922:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080256, portal 10 00000100:00000001:0.0:1788573206.338925:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573206.338926:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788573206.338934:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.338942:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573206.338945:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573206.338947:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573206.338950:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810080256, offset 0 00000400:00000200:0.0:1788573206.338954:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573206.338958:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.338961:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573206.338963:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573206.338965:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573206.338967:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573206.338969:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573206.338971:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540743:540743:256:4294967295] 192.168.202.148@tcp LPNI seq info [540708:540708:8:4294967295] 00000400:00000200:0.0:1788573206.338974:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.338979:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573206.338982:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.338984:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573206.338987:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573206.338989:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 25 secs later) 00000800:00000100:0.0:1788573206.338997:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.338998:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573206.339001:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573206.339003:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573206.339006:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573206.339007:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573206.339013:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573206.339016:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339022:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339024:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339028:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339030:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573206.339032:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573206.339035:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573206.339037:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573206.339039:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308345369 00000100:00000001:0.0:1788573206.339041:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339043:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308345369 00000100:00000001:0.0:1788573206.339045:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339046:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573206.339047:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339049:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573206.339051:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573206/real 1788573206] req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573206.339056:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573206.339057:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573206.339058:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573206.339059:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573206.339060:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573206.339062:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573206.339064:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573206.339069:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339080:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573206.339081:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339082:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339083:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573206.339086:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339090:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573206.339093:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339098:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573206.339099:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339100:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339101:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339102:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573206.339103:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573206.339105:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573206.339107:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573206.339108:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573206.339110:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573206.339115:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080256:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573206.339118:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573206.339120:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339124:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.339125:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339130:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573206.339131:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573206.339133:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.339135:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080384:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573206.339137:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573206.339139:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573206.339140:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339141:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339143:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573206.339144:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.339147:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573206.339148:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573206.339151:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573206.339153:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080384, portal 10 00000100:00000001:0.0:1788573206.339154:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573206.339155:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788573206.339159:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339163:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573206.339165:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573206.339166:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573206.339167:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810080384, offset 0 00000400:00000200:0.0:1788573206.339170:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339172:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339175:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573206.339176:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573206.339178:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573206.339179:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573206.339181:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339183:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540744:540744:256:4294967295] 192.168.202.148@tcp LPNI seq info [540709:540709:8:4294967295] 00000400:00000200:0.0:1788573206.339186:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339190:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573206.339196:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339199:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573206.339201:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573206.339203:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 25 secs later) 00000800:00000100:0.0:1788573206.339210:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339212:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573206.339214:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573206.339216:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573206.339218:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573206.339219:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573206.339221:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573206.339223:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339228:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339230:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339234:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339235:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573206.339237:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573206.339240:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573206.339242:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573206.339243:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308345373 00000100:00000001:0.0:1788573206.339244:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339246:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308345373 00000100:00000001:0.0:1788573206.339247:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339249:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573206.339250:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339251:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573206.339253:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573206/real 1788573206] req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573206.339262:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573206.339263:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573206.339264:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573206.339265:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573206.339266:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573206.339267:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573206.339270:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573206.339276:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339280:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573206.339281:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339282:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339282:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573206.339284:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339288:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573206.339291:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339295:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573206.339296:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339297:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339298:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339299:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573206.339300:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573206.339302:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573206.339303:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573206.339305:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573206.339312:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573206.339318:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080384:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573206.339320:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573206.339322:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339326:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.339327:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339328:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573206.339329:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573206.339330:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.339332:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080512:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573206.339334:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573206.339336:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573206.339337:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339338:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339340:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573206.339341:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.339359:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573206.339361:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573206.339364:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573206.339366:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080512, portal 4 00000100:00000001:0.0:1788573206.339368:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573206.339369:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788573206.339373:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339377:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573206.339378:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573206.339380:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573206.339381:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810080512, offset 0 00000400:00000200:0.0:1788573206.339388:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339391:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339393:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573206.339395:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573206.339397:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573206.339399:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573206.339400:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339402:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540745:540745:256:4294967295] 192.168.202.148@tcp LPNI seq info [540710:540710:8:4294967295] 00000400:00000200:0.0:1788573206.339405:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339409:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573206.339412:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339414:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573206.339416:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573206.339418:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 25 secs later) 00000800:00000100:0.0:1788573206.339426:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339427:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573206.339429:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573206.339431:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573206.339433:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573206.339434:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573206.339436:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573206.339438:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339443:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339445:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339449:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339450:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573206.339452:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573206.339454:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573206.339456:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573206.339461:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308345505 00000100:00000001:0.0:1788573206.339462:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339464:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308345505 00000100:00000001:0.0:1788573206.339465:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339466:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573206.339468:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339469:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573206.339471:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573206/real 1788573206] req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573206.339477:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573206.339477:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573206.339479:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573206.339480:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573206.339481:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573206.339483:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573206.339485:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573206.339490:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339494:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573206.339495:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339496:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339497:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573206.339499:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339503:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573206.339505:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339509:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573206.339513:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339514:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339515:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339516:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573206.339518:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573206.339520:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573206.339522:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573206.339523:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573206.339526:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573206.339531:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080512:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573206.339533:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339534:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339536:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440810080256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339540:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573206.339541:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339542:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573206.339544:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573206.339545:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573206.339546:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=3 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573206.339547:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573206.339549:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788573206.339551:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573206.339552:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573206.339553:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788573206.339555:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573206.339556:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339557:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339559:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440810080384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339569:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573206.339570:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339572:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573206.339573:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573206.339574:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573206.339575:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573206.339576:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573206.339577:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d400. 02000000:00000001:0.0:1788573206.339578:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573206.339579:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573206.339579:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788573206.339581:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573206.339582:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339583:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339585:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810080512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339588:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573206.339589:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339590:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573206.339591:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573206.339592:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573206.339592:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573206.339593:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573206.339594:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233dc00. 02000000:00000001:0.0:1788573206.339595:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573206.339596:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573206.339597:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788573206.339598:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573206.339599:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339600:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339602:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573206.339603:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339604:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573206.339605:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573206.339606:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573206.339611:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339616:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.339617:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339619:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573206.339620:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573206.339621:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.339623:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080640:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573206.339625:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573206.339626:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573206.339627:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573206.339629:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339630:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573206.339631:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573206.339634:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573206.339651:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573206.339654:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573206.339656:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080640, portal 4 00000100:00000001:0.0:1788573206.339658:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573206.339659:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291922176 : -109714417629440 : ffff9c371f1c4700) 00000100:00000040:0.0:1788573206.339663:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339667:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573206.339669:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573206.339670:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573206.339672:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810080640, offset 0 00000400:00000200:0.0:1788573206.339674:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339677:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339679:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573206.339681:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573206.339683:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573206.339685:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573206.339686:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573206.339692:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540746:540746:256:4294967295] 192.168.202.148@tcp LPNI seq info [540711:540711:8:4294967295] 00000400:00000200:0.0:1788573206.339695:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573206.339700:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573206.339703:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339705:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573206.339707:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573206.339709:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 25 secs later) 00000800:00000100:0.0:1788573206.339717:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573206.339718:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573206.339720:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573206.339722:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573206.339724:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573206.339725:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573206.339727:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573206.339730:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339735:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339737:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339742:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339743:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573206.339744:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573206.339746:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573206.339749:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573206.339750:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308345515 00000100:00000001:0.0:1788573206.339752:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339753:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308345515 00000100:00000001:0.0:1788573206.339755:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308349150 00000100:00000001:0.0:1788573206.339756:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573206.339761:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339762:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573206.339765:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573206/real 1788573206] req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573206.339771:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573206.339772:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573206.339774:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573206.339775:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573206.339776:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573206.339777:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573206.339779:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573206.339784:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339789:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573206.339790:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339791:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339792:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573206.339794:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339798:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573206.339800:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339804:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573206.339806:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339807:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573206.339807:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339808:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573206.339809:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573206.339811:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573206.339816:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573206.339817:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573206.339820:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573206.339825:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810080640:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573206.339827:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339828:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573206.339830:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4700 x1875440810080640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573206.339834:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573206.339835:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573206.339837:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573206.339838:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573206.339839:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573206.339840:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573206.339841:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573206.339842:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d800. 02000000:00000001:0.0:1788573206.339844:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573206.339845:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573206.339846:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4700. 00000100:00000001:0.0:1788573206.339847:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573206.339848:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339849:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.339852:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573206.339853:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.339854:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573206.339855:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573207.745777:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573207.745780:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573207.745784:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573210.177781:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308349150, at 4308349184 00000100:00000010:0.0:1788573210.177805:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573211.458003:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573211.458015:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573211.458023:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573211.458043:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.458084:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.458092:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.458103:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573211.458109:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573211.458121:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.458135:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810080896:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573211.458152:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573211.458159:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573211.458165:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.458176:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573211.458191:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573211.458199:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.458218:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573211.458231:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573211.458266:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573211.458281:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810080896, portal 25 00000100:00000001:0.0:1788573211.458293:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573211.458300:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286336128 : -109714423215488 : ffff9c371ec70a80) 00000100:00000040:0.0:1788573211.458322:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.458350:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573211.458364:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573211.458373:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573211.458388:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810080896, offset 0 00000400:00000200:0.0:1788573211.458412:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573211.458429:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.458447:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573211.458465:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573211.458499:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573211.458507:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573211.458518:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573211.458538:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540747:540747:256:4294967295] 192.168.202.148@tcp LPNI seq info [540712:540712:8:4294967295] 00000400:00000200:0.0:1788573211.458559:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.458594:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573211.458615:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.458629:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573211.458698:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573211.458711:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573211.458765:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.458776:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573211.458793:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573211.458813:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573211.458830:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573211.458840:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573211.458852:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573211.458866:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.458909:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.458920:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.458963:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.458967:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573211.458977:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.458988:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573211.458994:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308354271 00000100:00000010:0.0:1788573211.459002:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573211.459014:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573211.459028:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308354271 00000100:00000010:0.0:1788573211.459047:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116262a0. 00000100:00000001:0.0:1788573211.459056:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573211.459063:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573211.459068:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459071:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573211.459080:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573211/real 1788573211] req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573211.459093:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573211.459095:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573211.459098:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573211.459102:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573211.459104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573211.459106:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573211.459111:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573211.459119:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459127:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573211.459130:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459132:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.459134:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573211.459139:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459147:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573211.459152:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459160:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573211.459163:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459165:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.459166:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459177:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573211.459179:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573211.459183:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573211.459189:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573211.459206:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573211.459212:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573211.459223:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810080896:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573211.459233:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459236:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.459240:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70a80 x1875440810080896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459261:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573211.459265:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573211.459269:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573211.459273:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573211.459276:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573211.459279:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573211.459281:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573211.459284:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788573211.459289:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573211.459291:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573211.459294:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70a80. 00000100:00000001:0.0:1788573211.459297:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573211.459299:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459302:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459311:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573211.459314:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459317:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573211.459319:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573211.459321:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573211.459325:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459346:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.459348:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.459351:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573211.459353:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573211.459356:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.459360:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081024:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573211.459364:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573211.459367:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573211.459368:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.459370:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573211.459373:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573211.459376:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.459380:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573211.459383:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573211.459387:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573211.459391:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081024, portal 10 00000100:00000001:0.0:1788573211.459393:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573211.459395:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573211.459403:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459409:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573211.459412:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573211.459414:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573211.459417:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810081024, offset 0 00000400:00000200:0.0:1788573211.459422:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573211.459426:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.459430:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573211.459433:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573211.459436:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573211.459439:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573211.459441:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573211.459450:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540748:540748:256:4294967295] 192.168.202.148@tcp LPNI seq info [540713:540713:8:4294967295] 00000400:00000200:0.0:1788573211.459454:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.459461:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573211.459467:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.459470:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573211.459474:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573211.459478:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573211.459490:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.459492:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573211.459496:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573211.459500:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573211.459503:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573211.459505:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573211.459508:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573211.459512:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459520:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.459524:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459531:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459532:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573211.459536:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573211.459541:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573211.459545:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573211.459548:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308350489 00000100:00000001:0.0:1788573211.459551:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.459553:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308350489 00000100:00000001:0.0:1788573211.459556:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.459558:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573211.459566:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459568:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573211.459573:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573211/real 1788573211] req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573211.459582:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573211.459583:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573211.459585:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573211.459588:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573211.459590:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573211.459592:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573211.459596:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573211.459605:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459612:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573211.459613:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459615:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.459617:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573211.459620:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459628:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573211.459632:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459666:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573211.459668:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459670:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.459672:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459674:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573211.459676:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573211.459679:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573211.459689:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573211.459691:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573211.459697:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573211.459705:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081024:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573211.459709:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573211.459713:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459720:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.459722:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.459725:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573211.459727:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573211.459729:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.459732:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081152:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573211.459736:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573211.459738:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573211.459740:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.459742:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573211.459745:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573211.459748:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.459751:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573211.459754:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573211.459757:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573211.459761:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081152, portal 10 00000100:00000001:0.0:1788573211.459763:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573211.459765:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286338816 : -109714423212800 : ffff9c371ec71500) 00000100:00000040:0.0:1788573211.459771:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459778:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573211.459789:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573211.459791:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573211.459794:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810081152, offset 0 00000400:00000200:0.0:1788573211.459799:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573211.459803:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.459807:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573211.459811:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573211.459813:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573211.459816:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573211.459818:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573211.459822:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540749:540749:256:4294967295] 192.168.202.148@tcp LPNI seq info [540714:540714:8:4294967295] 00000400:00000200:0.0:1788573211.459826:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.459833:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573211.459838:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.459842:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573211.459846:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573211.459849:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573211.459861:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.459863:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573211.459866:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573211.459869:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573211.459873:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573211.459874:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573211.459877:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573211.459881:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459890:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.459893:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459900:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459907:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573211.459912:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573211.459916:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573211.459920:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573211.459922:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308350494 00000100:00000001:0.0:1788573211.459924:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.459927:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308350494 00000100:00000001:0.0:1788573211.459929:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.459932:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573211.459934:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.459936:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573211.459940:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573211/real 1788573211] req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573211.459949:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573211.459950:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573211.459952:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573211.459955:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573211.459957:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573211.459959:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573211.459962:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573211.459971:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459978:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573211.459979:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.459981:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.459982:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573211.459986:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.459993:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573211.460003:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460011:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573211.460013:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460015:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.460017:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460019:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573211.460021:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573211.460023:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573211.460027:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573211.460029:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573211.460033:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573211.460041:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081152:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573211.460046:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573211.460050:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460057:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.460058:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.460061:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573211.460063:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573211.460065:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.460068:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081280:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573211.460072:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573211.460074:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573211.460075:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.460077:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460081:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573211.460083:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.460094:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573211.460097:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573211.460102:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573211.460105:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081280, portal 4 00000100:00000001:0.0:1788573211.460107:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573211.460108:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286335232 : -109714423216384 : ffff9c371ec70700) 00000100:00000040:0.0:1788573211.460115:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460122:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573211.460124:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573211.460126:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573211.460129:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810081280, offset 0 00000400:00000200:0.0:1788573211.460133:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573211.460137:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.460141:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573211.460144:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573211.460146:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573211.460149:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573211.460152:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573211.460155:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540750:540750:256:4294967295] 192.168.202.148@tcp LPNI seq info [540715:540715:8:4294967295] 00000400:00000200:0.0:1788573211.460159:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.460165:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573211.460169:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.460173:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573211.460177:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573211.460180:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573211.460190:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.460193:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573211.460196:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573211.460199:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573211.460202:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573211.460210:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573211.460214:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573211.460218:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460227:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460230:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460236:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.460238:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573211.460242:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573211.460246:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573211.460264:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573211.460266:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308350626 00000100:00000001:0.0:1788573211.460269:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.460271:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308350626 00000100:00000001:0.0:1788573211.460274:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.460276:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573211.460278:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.460280:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573211.460284:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573211/real 1788573211] req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573211.460293:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573211.460295:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573211.460297:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573211.460300:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573211.460302:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573211.460304:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573211.460307:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573211.460316:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460330:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573211.460332:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460334:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.460336:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573211.460340:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460347:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573211.460352:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460360:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573211.460362:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460364:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.460366:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460368:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573211.460371:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573211.460374:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573211.460377:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573211.460380:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573211.460384:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573211.460393:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081280:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573211.460397:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573211.460401:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460408:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.460410:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.460419:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573211.460421:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573211.460423:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.460427:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081408:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573211.460430:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573211.460432:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573211.460434:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573211.460436:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460439:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573211.460442:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573211.460445:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573211.460448:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573211.460452:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573211.460456:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081408, portal 4 00000100:00000001:0.0:1788573211.460458:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573211.460459:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345088 : -109714423206528 : ffff9c371ec72d80) 00000100:00000040:0.0:1788573211.460466:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460473:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573211.460476:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573211.460478:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573211.460480:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810081408, offset 0 00000400:00000200:0.0:1788573211.460485:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573211.460489:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.460493:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573211.460496:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573211.460498:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573211.460501:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573211.460504:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573211.460507:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540751:540751:256:4294967295] 192.168.202.148@tcp LPNI seq info [540716:540716:8:4294967295] 00000400:00000200:0.0:1788573211.460511:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573211.460518:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573211.460549:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.460553:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573211.460557:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573211.460560:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 20 secs later) 00000800:00000100:0.0:1788573211.460571:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573211.460573:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573211.460577:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573211.460581:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573211.460584:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573211.460586:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573211.460590:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573211.460594:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460603:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460606:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460612:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.460614:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573211.460618:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573211.460622:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573211.460625:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573211.460627:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308350635 00000100:00000001:0.0:1788573211.460630:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.460632:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308350635 00000100:00000001:0.0:1788573211.460651:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308354271 00000100:00000001:0.0:1788573211.460655:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573211.460657:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.460659:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573211.460663:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573211/real 1788573211] req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573211.460678:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573211.460680:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573211.460682:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573211.460685:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573211.460687:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573211.460689:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573211.460693:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573211.460702:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460709:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573211.460711:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460713:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.460714:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573211.460718:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460725:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573211.460730:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460738:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573211.460740:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460742:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573211.460744:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460746:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573211.460748:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573211.460751:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573211.460755:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573211.460757:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573211.460768:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573211.460777:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081408:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573211.460781:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460784:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460787:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810081024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460795:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573211.460796:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460798:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573211.460801:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573211.460803:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573211.460804:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573211.460807:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573211.460808:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573211.460811:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573211.460813:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573211.460815:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573211.460818:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573211.460819:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460821:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460825:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71500 x1875440810081152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460832:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573211.460833:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460835:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573211.460838:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573211.460839:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573211.460840:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573211.460842:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573211.460844:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573211.460846:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573211.460853:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573211.460855:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71500. 00000100:00000001:0.0:1788573211.460858:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573211.460859:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460861:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460865:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70700 x1875440810081280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460872:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573211.460873:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460875:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573211.460878:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573211.460879:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573211.460881:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573211.460883:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573211.460884:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573211.460887:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573211.460888:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573211.460890:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70700. 00000100:00000001:0.0:1788573211.460892:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573211.460894:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460896:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573211.460900:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72d80 x1875440810081408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573211.460906:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573211.460908:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573211.460909:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573211.460912:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573211.460913:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573211.460914:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573211.460916:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573211.460918:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573211.460920:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573211.460921:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573211.460923:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72d80. 00000100:00000001:0.0:1788573211.460926:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573211.460927:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460934:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.460938:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573211.460940:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.460942:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573211.460943:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573212.865834:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573212.865838:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573212.865841:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573215.297699:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308354271, at 4308354304 00000100:00000010:0.0:1788573215.297713:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573216.578010:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573216.578014:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573216.578018:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573216.578026:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578046:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.578052:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.578060:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573216.578080:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573216.578089:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.578098:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810081536:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573216.578107:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573216.578115:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573216.578121:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.578129:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573216.578139:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573216.578147:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.578187:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573216.578192:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573216.578208:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573216.578217:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081536, portal 25 00000100:00000001:0.0:1788573216.578222:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573216.578225:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573216.578251:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578273:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573216.578280:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573216.578285:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573216.578290:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810081536, offset 0 00000400:00000200:0.0:1788573216.578300:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573216.578310:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.578319:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573216.578341:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573216.578347:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573216.578353:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573216.578357:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573216.578365:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540752:540752:256:4294967295] 192.168.202.148@tcp LPNI seq info [540717:540717:8:4294967295] 00000400:00000200:0.0:1788573216.578375:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.578391:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573216.578398:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.578405:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573216.578414:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573216.578419:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573216.578465:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.578471:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573216.578480:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573216.578491:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573216.578500:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573216.578505:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573216.578513:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573216.578522:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578550:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.578555:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578585:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.578589:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573216.578598:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.578606:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573216.578609:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308359390 00000100:00000010:0.0:1788573216.578615:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573216.578625:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573216.578634:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308359390 00000100:00000010:0.0:1788573216.578668:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626840. 00000100:00000001:0.0:1788573216.578673:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573216.578680:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573216.578684:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.578698:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573216.578704:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573216/real 1788573216] req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573216.578723:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573216.578727:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573216.578733:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573216.578740:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573216.578745:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573216.578753:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573216.578763:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573216.578789:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578835:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573216.578840:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.578845:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.578860:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573216.578870:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578895:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573216.578903:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.578920:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573216.578925:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.578928:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.578930:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.578932:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573216.578936:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573216.578939:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573216.578966:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573216.578989:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573216.578998:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573216.579009:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810081536:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573216.579014:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579019:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.579022:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810081536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579028:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573216.579030:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573216.579033:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573216.579036:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573216.579037:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573216.579039:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573216.579057:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573216.579059:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573216.579063:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573216.579077:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573216.579079:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573216.579082:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573216.579083:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579084:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579090:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573216.579091:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579093:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573216.579094:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573216.579096:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573216.579099:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579104:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.579105:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579107:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573216.579109:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573216.579110:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.579112:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573216.579114:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573216.579116:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573216.579117:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579119:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573216.579121:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573216.579122:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.579126:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573216.579128:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573216.579131:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573216.579134:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081664, portal 10 00000100:00000001:0.0:1788573216.579135:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573216.579136:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573216.579141:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579146:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573216.579148:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573216.579172:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573216.579174:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810081664, offset 0 00000400:00000200:0.0:1788573216.579178:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579181:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579185:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573216.579204:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573216.579206:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573216.579208:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573216.579210:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579212:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540753:540753:256:4294967295] 192.168.202.148@tcp LPNI seq info [540718:540718:8:4294967295] 00000400:00000200:0.0:1788573216.579215:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579220:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573216.579223:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579226:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573216.579229:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573216.579231:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573216.579239:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579241:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573216.579243:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573216.579245:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573216.579248:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573216.579249:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573216.579251:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573216.579254:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579260:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.579262:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579267:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579268:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573216.579274:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573216.579278:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573216.579280:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573216.579282:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308355609 00000100:00000001:0.0:1788573216.579284:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579286:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308355609 00000100:00000001:0.0:1788573216.579287:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579288:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573216.579290:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579291:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573216.579294:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573216/real 1788573216] req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573216.579300:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573216.579301:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573216.579302:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573216.579304:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573216.579305:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573216.579306:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573216.579308:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573216.579322:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579329:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573216.579330:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579331:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579332:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573216.579335:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579340:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573216.579342:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579367:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573216.579369:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579370:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579371:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579372:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573216.579373:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573216.579375:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573216.579377:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573216.579378:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573216.579381:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573216.579386:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573216.579389:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573216.579391:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579395:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.579396:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579397:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573216.579399:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573216.579400:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.579402:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081792:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573216.579403:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573216.579405:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573216.579406:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579407:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573216.579410:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573216.579412:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.579419:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573216.579422:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573216.579426:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573216.579429:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081792, portal 10 00000100:00000001:0.0:1788573216.579431:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573216.579432:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573216.579439:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579445:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573216.579448:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573216.579450:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573216.579452:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810081792, offset 0 00000400:00000200:0.0:1788573216.579467:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579471:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579475:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573216.579478:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573216.579480:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573216.579483:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573216.579485:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579488:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540754:540754:256:4294967295] 192.168.202.148@tcp LPNI seq info [540719:540719:8:4294967295] 00000400:00000200:0.0:1788573216.579493:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579499:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573216.579504:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579507:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573216.579511:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573216.579515:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573216.579529:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579539:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573216.579541:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573216.579544:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573216.579547:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573216.579550:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573216.579553:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573216.579556:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579561:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.579563:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579568:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579569:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573216.579571:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573216.579574:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573216.579576:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573216.579578:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308355613 00000100:00000001:0.0:1788573216.579579:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579581:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308355613 00000100:00000001:0.0:1788573216.579582:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579584:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573216.579585:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579586:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573216.579588:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573216/real 1788573216] req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573216.579594:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573216.579594:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573216.579596:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573216.579597:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573216.579599:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573216.579600:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573216.579602:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573216.579607:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579615:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573216.579616:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579617:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579618:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573216.579620:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579625:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573216.579627:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579632:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573216.579633:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579634:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579635:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579636:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573216.579637:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573216.579639:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573216.579641:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573216.579642:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573216.579644:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573216.579649:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081792:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573216.579668:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573216.579670:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579675:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.579676:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579683:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573216.579684:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573216.579686:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.579688:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573216.579690:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573216.579692:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573216.579693:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579694:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573216.579696:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573216.579697:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.579700:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573216.579702:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573216.579705:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573216.579707:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810081920, portal 4 00000100:00000001:0.0:1788573216.579708:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573216.579709:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573216.579713:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579717:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573216.579719:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573216.579720:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573216.579722:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810081920, offset 0 00000400:00000200:0.0:1788573216.579725:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579727:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579729:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573216.579731:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573216.579733:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573216.579734:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573216.579736:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579738:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540755:540755:256:4294967295] 192.168.202.148@tcp LPNI seq info [540720:540720:8:4294967295] 00000400:00000200:0.0:1788573216.579741:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579746:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573216.579752:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579754:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573216.579757:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573216.579759:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573216.579766:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.579768:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573216.579770:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573216.579772:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573216.579774:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573216.579775:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573216.579777:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573216.579780:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579785:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.579787:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579808:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579809:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573216.579812:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573216.579814:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573216.579816:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573216.579818:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308355746 00000100:00000001:0.0:1788573216.579820:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579821:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308355746 00000100:00000001:0.0:1788573216.579823:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.579824:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573216.579825:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.579826:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573216.579829:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573216/real 1788573216] req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573216.579838:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573216.579839:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573216.579840:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573216.579842:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573216.579842:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573216.579844:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573216.579846:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573216.579852:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579856:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573216.579857:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579858:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579859:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573216.579861:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579865:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573216.579868:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579873:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573216.579875:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579876:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.579876:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.579878:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573216.579879:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573216.579881:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573216.579883:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573216.579884:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573216.579898:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573216.579903:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810081920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573216.579906:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573216.579908:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579912:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.579913:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579915:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573216.579916:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573216.579917:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.579919:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082048:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573216.579921:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573216.579922:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573216.579923:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573216.579924:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573216.579926:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573216.579928:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573216.579930:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573216.579931:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573216.579934:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573216.579936:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082048, portal 4 00000100:00000001:0.0:1788573216.579937:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573216.579938:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788573216.579942:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.579947:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573216.579949:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573216.579950:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573216.579978:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810082048, offset 0 00000400:00000200:0.0:1788573216.579984:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579986:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.579989:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573216.579991:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573216.579992:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573216.579994:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573216.579995:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573216.579997:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540756:540756:256:4294967295] 192.168.202.148@tcp LPNI seq info [540721:540721:8:4294967295] 00000400:00000200:0.0:1788573216.580000:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573216.580004:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573216.580007:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.580009:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573216.580011:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573216.580012:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 15 secs later) 00000800:00000100:0.0:1788573216.580019:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573216.580021:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573216.580023:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573216.580025:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573216.580027:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573216.580043:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573216.580045:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573216.580047:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580052:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.580054:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580058:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.580059:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573216.580061:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573216.580063:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573216.580065:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573216.580069:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308355755 00000100:00000001:0.0:1788573216.580071:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.580073:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308355755 00000100:00000001:0.0:1788573216.580074:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308359390 00000100:00000001:0.0:1788573216.580076:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573216.580077:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.580078:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573216.580081:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573216/real 1788573216] req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573216.580086:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573216.580087:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573216.580088:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573216.580089:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573216.580091:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573216.580108:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573216.580122:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573216.580132:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580140:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573216.580142:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580145:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.580146:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573216.580162:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580187:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573216.580191:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580209:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573216.580220:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580221:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573216.580223:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580224:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573216.580226:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573216.580229:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573216.580233:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573216.580236:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573216.580240:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573216.580248:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082048:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573216.580252:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580254:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.580257:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810081664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580264:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573216.580265:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573216.580267:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573216.580270:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573216.580272:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573216.580273:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573216.580276:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573216.580278:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573216.580280:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573216.580281:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573216.580283:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573216.580286:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573216.580287:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580289:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.580293:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810081792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580307:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573216.580308:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573216.580309:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573216.580311:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573216.580312:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573216.580313:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573216.580315:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573216.580316:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788573216.580317:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573216.580318:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573216.580319:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573216.580320:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573216.580321:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580322:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.580325:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810081920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580328:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573216.580329:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573216.580330:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573216.580331:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573216.580332:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573216.580333:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573216.580334:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573216.580335:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788573216.580337:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573216.580337:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573216.580338:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573216.580339:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573216.580340:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580341:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573216.580343:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810082048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573216.580346:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573216.580347:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573216.580351:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788573216.580352:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573216.580353:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573216.580354:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573216.580355:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573216.580356:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788573216.580358:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573216.580358:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573216.580359:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573216.580361:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573216.580361:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580363:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.580365:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573216.580367:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.580368:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573216.580370:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573217.985786:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573217.985790:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573217.985793:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573220.417770:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308359390, at 4308359424 00000100:00000010:0.0:1788573220.417791:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000800:00000200:0.0:1788573221.121991:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573221.697970:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573221.697978:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573221.697983:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573221.698007:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698022:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.698025:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.698032:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573221.698036:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573221.698041:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.698048:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810082176:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573221.698087:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573221.698094:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573221.698105:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.698110:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573221.698117:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573221.698121:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.698128:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573221.698131:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573221.698140:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573221.698149:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082176, portal 25 00000100:00000001:0.0:1788573221.698152:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573221.698154:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788573221.698161:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698172:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573221.698179:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573221.698185:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573221.698189:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810082176, offset 0 00000400:00000200:0.0:1788573221.698199:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573221.698218:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.698227:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573221.698235:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573221.698245:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573221.698253:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573221.698258:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573221.698264:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540757:540757:256:4294967295] 192.168.202.148@tcp LPNI seq info [540722:540722:8:4294967295] 00000400:00000200:0.0:1788573221.698272:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.698284:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573221.698294:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.698301:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573221.698310:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573221.698315:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1788573221.698343:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.698355:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573221.698361:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573221.698371:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573221.698378:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573221.698382:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573221.698389:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573221.698396:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698425:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.698431:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698444:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.698448:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573221.698453:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.698461:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573221.698464:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308364510 00000100:00000010:0.0:1788573221.698470:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573221.698478:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573221.698484:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308364510 00000100:00000010:0.0:1788573221.698489:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116266c0. 00000100:00000001:0.0:1788573221.698493:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573221.698498:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573221.698502:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.698506:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573221.698512:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573221/real 1788573221] req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573221.698535:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573221.698539:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573221.698544:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573221.698551:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573221.698555:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573221.698561:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573221.698574:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573221.698591:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698609:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573221.698612:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698616:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.698620:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573221.698629:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698682:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573221.698689:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698703:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573221.698710:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698715:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.698718:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698723:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573221.698736:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573221.698744:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573221.698755:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573221.698760:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573221.698771:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573221.698787:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810082176:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573221.698795:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698798:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.698805:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810082176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698826:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573221.698830:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573221.698835:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573221.698839:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573221.698842:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573221.698845:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573221.698848:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573221.698851:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788573221.698855:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573221.698858:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573221.698860:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573221.698867:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573221.698869:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698873:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.698880:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573221.698883:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.698886:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573221.698889:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573221.698891:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573221.698896:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.698909:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.698911:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.698916:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573221.698918:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573221.698923:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.698929:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082304:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573221.698935:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573221.698939:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573221.698942:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.698946:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573221.698951:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573221.698957:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.698967:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573221.698979:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573221.698988:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573221.698994:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082304, portal 10 00000100:00000001:0.0:1788573221.698999:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573221.699003:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286341504 : -109714423210112 : ffff9c371ec71f80) 00000100:00000040:0.0:1788573221.699018:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699028:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573221.699030:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573221.699033:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573221.699036:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810082304, offset 0 00000400:00000200:0.0:1788573221.699039:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699042:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699045:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573221.699048:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573221.699051:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573221.699053:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573221.699069:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699073:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540758:540758:256:4294967295] 192.168.202.148@tcp LPNI seq info [540723:540723:8:4294967295] 00000400:00000200:0.0:1788573221.699077:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699085:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573221.699090:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699094:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573221.699099:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573221.699103:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1788573221.699114:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699115:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573221.699118:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573221.699120:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573221.699123:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573221.699124:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573221.699132:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573221.699135:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699141:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699143:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699148:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699149:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573221.699152:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573221.699154:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573221.699157:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573221.699158:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308360729 00000100:00000001:0.0:1788573221.699161:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699162:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308360729 00000100:00000001:0.0:1788573221.699164:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699165:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573221.699167:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699168:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573221.699170:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573221/real 1788573221] req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573221.699176:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573221.699177:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573221.699178:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573221.699180:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573221.699181:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573221.699182:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573221.699184:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573221.699189:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699201:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573221.699202:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699203:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699204:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573221.699207:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699211:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573221.699214:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699219:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573221.699220:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699221:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699222:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699223:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573221.699225:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573221.699226:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573221.699228:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573221.699229:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573221.699232:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573221.699237:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082304:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573221.699239:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573221.699242:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699246:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.699247:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699251:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573221.699253:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573221.699254:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.699256:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082432:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573221.699258:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573221.699260:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573221.699261:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699262:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699264:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573221.699266:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.699268:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573221.699270:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573221.699272:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573221.699274:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082432, portal 10 00000100:00000001:0.0:1788573221.699276:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573221.699277:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286344192 : -109714423207424 : ffff9c371ec72a00) 00000100:00000040:0.0:1788573221.699280:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699285:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573221.699286:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573221.699288:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573221.699289:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810082432, offset 0 00000400:00000200:0.0:1788573221.699292:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699294:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699297:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573221.699299:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573221.699301:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573221.699302:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573221.699304:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699306:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540759:540759:256:4294967295] 192.168.202.148@tcp LPNI seq info [540724:540724:8:4294967295] 00000400:00000200:0.0:1788573221.699309:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699313:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573221.699320:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699322:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573221.699325:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573221.699327:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1788573221.699334:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699336:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573221.699338:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573221.699340:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573221.699342:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573221.699343:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573221.699345:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573221.699347:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699352:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699354:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699359:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699360:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573221.699362:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573221.699365:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573221.699367:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573221.699368:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308360733 00000100:00000001:0.0:1788573221.699370:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699371:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308360733 00000100:00000001:0.0:1788573221.699372:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699374:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573221.699375:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699376:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573221.699378:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573221/real 1788573221] req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573221.699387:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573221.699388:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573221.699389:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573221.699390:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573221.699391:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573221.699393:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573221.699395:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573221.699400:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699404:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573221.699405:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699406:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699407:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573221.699409:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699413:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573221.699415:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699420:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573221.699421:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699423:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699423:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699425:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573221.699426:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573221.699427:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573221.699429:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573221.699430:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573221.699436:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573221.699441:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082432:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573221.699444:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573221.699446:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699450:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.699451:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699452:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573221.699454:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573221.699455:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.699457:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082560:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573221.699459:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573221.699460:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573221.699461:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699462:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699464:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573221.699466:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.699468:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573221.699470:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573221.699472:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573221.699475:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082560, portal 4 00000100:00000001:0.0:1788573221.699476:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573221.699477:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337920 : -109714423213696 : ffff9c371ec71180) 00000100:00000040:0.0:1788573221.699481:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699485:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573221.699486:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573221.699488:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573221.699489:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810082560, offset 0 00000400:00000200:0.0:1788573221.699495:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699497:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699500:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573221.699502:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573221.699503:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573221.699505:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573221.699507:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699509:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540760:540760:256:4294967295] 192.168.202.148@tcp LPNI seq info [540725:540725:8:4294967295] 00000400:00000200:0.0:1788573221.699512:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699516:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573221.699518:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699521:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573221.699525:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573221.699528:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1788573221.699538:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699540:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573221.699543:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573221.699547:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573221.699550:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573221.699552:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573221.699555:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573221.699559:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699567:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699570:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699577:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699578:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573221.699580:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573221.699583:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573221.699585:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573221.699592:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308360865 00000100:00000001:0.0:1788573221.699594:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699595:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308360865 00000100:00000001:0.0:1788573221.699597:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699598:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573221.699600:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699601:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573221.699604:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573221/real 1788573221] req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573221.699609:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573221.699610:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573221.699611:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573221.699613:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573221.699614:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573221.699616:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573221.699618:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573221.699626:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699633:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573221.699654:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699657:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699659:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573221.699664:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699671:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573221.699675:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699680:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573221.699686:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699688:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699688:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699690:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573221.699691:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573221.699693:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573221.699696:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573221.699697:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573221.699700:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573221.699705:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082560:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573221.699708:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699709:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699711:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810082304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699715:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573221.699716:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699718:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573221.699719:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573221.699720:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573221.699721:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573221.699723:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573221.699724:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788573221.699726:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573221.699727:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573221.699728:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788573221.699730:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573221.699731:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699732:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699734:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810082432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699745:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573221.699746:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699747:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573221.699749:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573221.699750:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573221.699750:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573221.699752:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573221.699752:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788573221.699754:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573221.699754:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573221.699755:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788573221.699757:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573221.699757:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699758:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699761:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71180 x1875440810082560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699764:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573221.699765:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699766:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573221.699767:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573221.699768:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573221.699768:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573221.699769:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573221.699770:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573221.699771:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573221.699772:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573221.699773:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71180. 00000100:00000001:0.0:1788573221.699774:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573221.699775:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699776:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699779:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573221.699780:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699781:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573221.699782:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573221.699783:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573221.699788:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699793:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.699794:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699795:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573221.699797:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573221.699798:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.699800:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082688:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573221.699802:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573221.699803:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573221.699805:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573221.699806:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573221.699808:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573221.699809:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573221.699811:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573221.699813:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573221.699815:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573221.699817:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082688, portal 4 00000100:00000001:0.0:1788573221.699819:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573221.699820:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788573221.699824:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699828:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573221.699830:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573221.699831:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573221.699833:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810082688, offset 0 00000400:00000200:0.0:1788573221.699836:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699838:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699840:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573221.699842:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573221.699844:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573221.699846:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573221.699848:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573221.699853:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540761:540761:256:4294967295] 192.168.202.148@tcp LPNI seq info [540726:540726:8:4294967295] 00000400:00000200:0.0:1788573221.699856:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573221.699861:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573221.699864:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699866:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573221.699869:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573221.699871:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1788573221.699878:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573221.699879:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573221.699881:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573221.699884:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573221.699886:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573221.699887:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573221.699889:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573221.699891:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699896:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.699898:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699903:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699904:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573221.699906:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573221.699908:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573221.699910:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573221.699911:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308360875 00000100:00000001:0.0:1788573221.699913:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699914:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308360875 00000100:00000001:0.0:1788573221.699916:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308364510 00000100:00000001:0.0:1788573221.699917:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573221.699921:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.699923:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573221.699925:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573221/real 1788573221] req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573221.699931:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573221.699932:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573221.699933:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573221.699935:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573221.699936:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573221.699937:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573221.699939:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573221.699944:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699948:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573221.699950:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699951:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699952:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573221.699955:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699962:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573221.699967:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.699974:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573221.699976:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699977:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573221.699979:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.699981:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573221.699983:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573221.699986:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573221.699995:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573221.699997:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573221.700002:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573221.700010:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810082688:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573221.700013:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.700014:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573221.700016:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810082688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573221.700020:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573221.700021:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573221.700023:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573221.700024:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573221.700025:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573221.700026:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573221.700028:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573221.700029:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573221.700031:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573221.700031:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573221.700032:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573221.700034:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573221.700035:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.700036:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.700039:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573221.700040:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.700041:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573221.700042:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573223.105817:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573223.105821:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573223.105825:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573223.234408:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573223.234414:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573223.234419:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573223.234437:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.234456:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.234461:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.234468:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573223.234471:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573223.234477:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.234484:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810082944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573223.234493:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573223.234498:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573223.234502:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.234508:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573223.234518:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573223.234525:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.234536:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573223.234543:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573223.234555:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573223.234565:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810082944, portal 25 00000100:00000001:0.0:1788573223.234570:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573223.234573:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291922176 : -109714417629440 : ffff9c371f1c4700) 00000100:00000040:0.0:1788573223.234582:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.234608:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573223.234614:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573223.234620:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573223.234625:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810082944, offset 0 00000400:00000200:0.0:1788573223.234633:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573223.234670:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.234680:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573223.234688:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573223.234693:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573223.234699:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573223.234703:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573223.234715:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540762:540762:256:4294967295] 192.168.202.148@tcp LPNI seq info [540727:540727:8:4294967295] 00000400:00000200:0.0:1788573223.234723:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.234736:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573223.234745:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.234753:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573223.234764:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573223.234769:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573223.234787:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.234791:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573223.234796:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573223.234815:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573223.234828:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573223.234832:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573223.234838:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573223.234846:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.234861:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.234866:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.234883:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.234886:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573223.234893:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.234901:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573223.234905:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308366047 00000100:00000001:0.0:1788573223.234913:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308364510 00000100:00000001:0.0:1788573223.234921:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308366047 00000100:00000001:0.0:1788573223.234929:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308364510 00000100:00000001:0.0:1788573223.234938:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573223.234946:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.234959:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573223.234970:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573223/real 1788573223] req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573223.235006:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573223.235011:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573223.235018:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573223.235024:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573223.235039:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573223.235042:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573223.235048:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573223.235064:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235079:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573223.235082:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235085:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235090:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573223.235098:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235114:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573223.235120:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235135:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573223.235139:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235147:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235149:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235153:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573223.235157:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573223.235162:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573223.235170:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573223.235189:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573223.235196:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573223.235210:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810082944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573223.235220:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235223:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.235227:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4700 x1875440810082944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235243:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573223.235247:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573223.235250:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573223.235255:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573223.235258:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573223.235262:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573223.235263:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573223.235266:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233dc00. 02000000:00000001:0.0:1788573223.235271:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573223.235273:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573223.235276:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4700. 00000100:00000001:0.0:1788573223.235280:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573223.235282:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235287:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235293:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573223.235294:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235296:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573223.235298:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573223.235299:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573223.235302:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235307:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.235308:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235310:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573223.235311:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573223.235317:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.235319:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573223.235322:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573223.235323:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573223.235324:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235325:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573223.235327:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573223.235329:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.235332:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573223.235333:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573223.235336:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573223.235339:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083072, portal 10 00000100:00000001:0.0:1788573223.235340:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573223.235341:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788573223.235346:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235350:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573223.235352:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573223.235353:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573223.235355:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810083072, offset 0 00000400:00000200:0.0:1788573223.235357:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235360:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235363:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573223.235365:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573223.235367:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573223.235368:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573223.235370:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235373:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540763:540763:256:4294967295] 192.168.202.148@tcp LPNI seq info [540728:540728:8:4294967295] 00000400:00000200:0.0:1788573223.235375:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235380:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573223.235384:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235389:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573223.235392:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573223.235394:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573223.235402:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235403:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573223.235405:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573223.235408:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573223.235410:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573223.235412:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573223.235413:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573223.235416:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235421:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.235423:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235428:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235429:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573223.235431:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573223.235434:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573223.235436:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573223.235437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308362265 00000100:00000001:0.0:1788573223.235439:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235440:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308362265 00000100:00000001:0.0:1788573223.235442:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235444:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573223.235445:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235446:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573223.235449:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573223/real 1788573223] req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573223.235462:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573223.235463:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573223.235464:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573223.235465:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573223.235466:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573223.235467:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573223.235470:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573223.235475:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235479:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573223.235480:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235481:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235482:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573223.235484:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235488:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573223.235491:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235495:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573223.235497:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235498:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235499:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235500:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573223.235501:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573223.235503:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573223.235505:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573223.235506:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573223.235511:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573223.235517:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573223.235521:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573223.235523:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235527:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.235528:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235530:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573223.235531:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573223.235533:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.235535:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573223.235537:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573223.235538:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573223.235539:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235540:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573223.235542:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573223.235544:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.235546:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573223.235548:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573223.235551:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573223.235554:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083200, portal 10 00000100:00000001:0.0:1788573223.235556:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573223.235557:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788573223.235561:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235565:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573223.235567:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573223.235568:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573223.235570:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810083200, offset 0 00000400:00000200:0.0:1788573223.235575:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235578:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235581:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573223.235583:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573223.235584:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573223.235586:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573223.235588:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235590:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540764:540764:256:4294967295] 192.168.202.148@tcp LPNI seq info [540729:540729:8:4294967295] 00000400:00000200:0.0:1788573223.235593:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235597:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573223.235600:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235602:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573223.235604:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573223.235606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573223.235613:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235614:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573223.235616:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573223.235619:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573223.235621:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573223.235622:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573223.235623:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573223.235626:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235631:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.235633:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235658:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235659:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573223.235662:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573223.235665:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573223.235667:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573223.235673:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308362270 00000100:00000001:0.0:1788573223.235675:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235676:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308362270 00000100:00000001:0.0:1788573223.235678:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235679:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573223.235681:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235682:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573223.235685:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573223/real 1788573223] req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573223.235690:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573223.235691:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573223.235692:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573223.235694:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573223.235695:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573223.235696:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573223.235698:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573223.235704:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235708:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573223.235709:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235710:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235711:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573223.235713:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235717:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573223.235720:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235724:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573223.235729:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235730:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.235731:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.235733:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573223.235734:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573223.235736:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573223.235738:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573223.235739:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573223.235741:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573223.235748:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573223.235752:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573223.235755:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235763:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.235764:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235767:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573223.235769:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573223.235771:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.235775:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573223.235777:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573223.235779:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573223.235781:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.235783:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573223.235787:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573223.235789:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.235793:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573223.235795:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573223.235799:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573223.235802:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083328, portal 4 00000100:00000001:0.0:1788573223.235809:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573223.235811:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788573223.235817:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235825:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573223.235827:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573223.235829:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573223.235832:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810083328, offset 0 00000400:00000200:0.0:1788573223.235836:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235840:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235844:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573223.235847:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573223.235849:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573223.235852:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573223.235854:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573223.235857:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540765:540765:256:4294967295] 192.168.202.148@tcp LPNI seq info [540730:540730:8:4294967295] 00000400:00000200:0.0:1788573223.235861:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.235867:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573223.235872:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235875:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573223.235879:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573223.235882:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573223.235893:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.235895:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573223.235898:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573223.235901:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573223.235904:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573223.235906:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573223.235908:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573223.235912:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235933:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.235937:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.235944:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235945:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573223.235950:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573223.235953:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573223.235957:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573223.235959:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308362402 00000100:00000001:0.0:1788573223.235962:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235964:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308362402 00000100:00000001:0.0:1788573223.235966:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.235968:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573223.235970:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.235972:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573223.235977:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573223/real 1788573223] req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573223.235985:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573223.235986:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573223.235988:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573223.235990:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573223.235992:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573223.235995:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573223.235998:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573223.236007:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236014:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573223.236021:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236023:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.236025:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573223.236046:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236054:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573223.236058:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236065:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573223.236068:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236069:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.236071:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236073:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573223.236076:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573223.236078:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573223.236082:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573223.236084:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573223.236088:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573223.236096:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573223.236101:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573223.236104:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236111:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.236112:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.236115:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573223.236116:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573223.236119:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.236129:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573223.236132:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573223.236134:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573223.236136:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573223.236137:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573223.236141:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573223.236143:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573223.236147:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573223.236150:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573223.236154:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573223.236157:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083456, portal 4 00000100:00000001:0.0:1788573223.236159:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573223.236161:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788573223.236167:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236173:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573223.236175:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573223.236178:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573223.236180:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810083456, offset 0 00000400:00000200:0.0:1788573223.236183:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573223.236187:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.236191:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573223.236194:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573223.236197:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573223.236199:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573223.236202:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573223.236205:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540766:540766:256:4294967295] 192.168.202.148@tcp LPNI seq info [540731:540731:8:4294967295] 00000400:00000200:0.0:1788573223.236210:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573223.236216:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573223.236220:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.236223:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573223.236233:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573223.236236:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573223.236250:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573223.236252:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573223.236256:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573223.236260:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573223.236263:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573223.236265:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573223.236268:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573223.236273:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236280:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.236284:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236291:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.236293:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573223.236296:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573223.236299:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573223.236303:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573223.236305:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308362411 00000100:00000001:0.0:1788573223.236308:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.236310:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308362411 00000100:00000001:0.0:1788573223.236312:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308366047 00000100:00000001:0.0:1788573223.236315:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573223.236317:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.236319:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573223.236323:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573223/real 1788573223] req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573223.236331:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573223.236338:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573223.236340:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573223.236342:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573223.236344:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573223.236346:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573223.236350:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573223.236359:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236366:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573223.236368:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236370:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.236371:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573223.236375:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236382:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573223.236387:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236393:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573223.236395:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236397:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573223.236398:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236400:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573223.236402:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573223.236404:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573223.236407:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573223.236409:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573223.236414:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573223.236433:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573223.236437:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236439:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.236442:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810083072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236448:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573223.236450:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573223.236452:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573223.236454:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573223.236456:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573223.236457:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573223.236460:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573223.236462:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d400. 02000000:00000001:0.0:1788573223.236464:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573223.236466:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573223.236468:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788573223.236470:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573223.236471:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236473:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.236477:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440810083200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236483:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573223.236484:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573223.236486:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573223.236488:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573223.236489:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573223.236490:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573223.236492:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573223.236494:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788573223.236496:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573223.236497:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573223.236499:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788573223.236506:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573223.236508:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236509:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.236513:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440810083328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236519:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573223.236520:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573223.236522:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573223.236524:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573223.236526:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573223.236527:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573223.236529:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573223.236530:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788573223.236533:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573223.236534:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573223.236535:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788573223.236538:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573223.236539:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236541:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573223.236544:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440810083456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573223.236551:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573223.236552:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573223.236554:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573223.236556:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573223.236557:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573223.236558:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573223.236560:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573223.236561:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788573223.236563:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573223.236564:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573223.236566:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788573223.236568:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573223.236569:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236571:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.236575:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573223.236581:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.236583:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573223.236585:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573227.073712:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308366047, at 4308366080 00000100:00000010:0.0:1788573227.073724:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573228.225812:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573228.225816:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573228.225819:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573228.353997:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573228.354008:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573228.354011:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573228.354020:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354036:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.354039:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.354042:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573228.354048:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573228.354053:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.354059:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810083584:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573228.354063:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573228.354067:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573228.354071:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.354075:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573228.354081:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573228.354085:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.354096:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573228.354106:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573228.354119:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573228.354128:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083584, portal 25 00000100:00000001:0.0:1788573228.354135:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573228.354139:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573228.354151:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354168:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573228.354181:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573228.354184:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573228.354187:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810083584, offset 0 00000400:00000200:0.0:1788573228.354197:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573228.354209:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.354217:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573228.354224:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573228.354228:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573228.354235:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573228.354240:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573228.354247:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540767:540767:256:4294967295] 192.168.202.148@tcp LPNI seq info [540732:540732:8:4294967295] 00000400:00000200:0.0:1788573228.354254:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.354267:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573228.354276:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.354283:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573228.354292:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573228.354297:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573228.354315:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.354318:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573228.354325:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573228.354333:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573228.354339:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573228.354343:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573228.354348:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573228.354355:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354374:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.354378:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354400:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.354403:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573228.354417:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.354427:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573228.354431:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308371166 00000100:00000010:0.0:1788573228.354438:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573228.354445:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573228.354452:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308371166 00000100:00000010:0.0:1788573228.354459:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116263c0. 00000100:00000001:0.0:1788573228.354464:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573228.354470:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573228.354474:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.354483:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573228.354492:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573228/real 1788573228] req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573228.354511:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573228.354515:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573228.354519:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573228.354523:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573228.354524:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573228.354526:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573228.354533:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573228.354551:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354569:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573228.354571:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354576:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.354580:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573228.354586:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354600:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573228.354606:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354624:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573228.354629:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354632:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.354673:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354677:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573228.354680:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573228.354686:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573228.354694:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573228.354704:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573228.354714:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573228.354731:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810083584:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573228.354741:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354745:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.354750:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810083584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354761:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573228.354765:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573228.354768:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573228.354774:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573228.354777:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573228.354780:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573228.354783:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573228.354788:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573228.354793:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573228.354798:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573228.354801:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573228.354805:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573228.354807:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354810:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.354824:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573228.354829:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.354833:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573228.354838:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573228.354884:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573228.354891:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.354902:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.354904:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.354910:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573228.354912:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573228.354914:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.354922:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083712:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573228.354943:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573228.354947:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573228.354951:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.354957:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573228.354963:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573228.354969:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.354979:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573228.354985:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573228.354992:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573228.354997:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083712, portal 10 00000100:00000001:0.0:1788573228.355001:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573228.355002:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573228.355010:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355015:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573228.355017:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573228.355019:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573228.355021:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810083712, offset 0 00000400:00000200:0.0:1788573228.355024:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355028:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355036:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573228.355038:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573228.355040:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573228.355042:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573228.355043:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355046:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540768:540768:256:4294967295] 192.168.202.148@tcp LPNI seq info [540733:540733:8:4294967295] 00000400:00000200:0.0:1788573228.355049:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355055:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573228.355061:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355064:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573228.355068:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573228.355071:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573228.355083:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355085:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573228.355088:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573228.355092:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573228.355096:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573228.355098:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573228.355101:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573228.355105:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355114:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355116:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355121:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355122:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573228.355124:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573228.355128:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573228.355130:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573228.355131:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308367385 00000100:00000001:0.0:1788573228.355138:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355139:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308367385 00000100:00000001:0.0:1788573228.355141:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355142:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573228.355144:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355145:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573228.355148:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573228/real 1788573228] req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573228.355153:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573228.355154:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573228.355156:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573228.355157:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573228.355158:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573228.355160:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573228.355162:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573228.355167:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355172:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573228.355173:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355174:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355175:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573228.355177:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355181:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573228.355183:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355187:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573228.355189:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355193:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355194:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355195:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573228.355197:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573228.355198:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573228.355201:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573228.355202:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573228.355204:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573228.355210:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083712:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573228.355213:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573228.355215:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355219:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.355219:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355221:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573228.355222:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573228.355223:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.355225:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573228.355227:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573228.355228:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573228.355229:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355230:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355233:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573228.355234:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.355236:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573228.355238:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573228.355240:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573228.355242:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083840, portal 10 00000100:00000001:0.0:1788573228.355244:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573228.355247:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573228.355252:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355256:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573228.355258:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573228.355259:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573228.355261:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810083840, offset 0 00000400:00000200:0.0:1788573228.355263:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355266:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355268:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573228.355270:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573228.355272:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573228.355273:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573228.355275:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355277:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540769:540769:256:4294967295] 192.168.202.148@tcp LPNI seq info [540734:540734:8:4294967295] 00000400:00000200:0.0:1788573228.355280:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355284:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573228.355287:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355290:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573228.355292:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573228.355294:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573228.355301:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355303:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573228.355305:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573228.355307:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573228.355309:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573228.355310:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573228.355312:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573228.355314:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355325:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355328:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355332:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355333:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573228.355336:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573228.355338:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573228.355340:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573228.355342:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308367389 00000100:00000001:0.0:1788573228.355343:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355345:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308367389 00000100:00000001:0.0:1788573228.355346:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355348:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573228.355349:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355350:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573228.355352:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573228/real 1788573228] req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573228.355357:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573228.355357:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573228.355358:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573228.355360:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573228.355361:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573228.355362:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573228.355364:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573228.355369:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355373:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573228.355377:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355378:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355379:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573228.355382:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355386:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573228.355389:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355393:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573228.355395:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355396:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355398:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573228.355399:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573228.355400:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573228.355402:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573228.355403:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573228.355406:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573228.355410:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573228.355413:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573228.355415:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355420:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.355421:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355422:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573228.355423:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573228.355424:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.355430:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083968:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573228.355432:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573228.355434:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573228.355435:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355436:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355438:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573228.355439:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.355442:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573228.355443:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573228.355445:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573228.355447:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810083968, portal 4 00000100:00000001:0.0:1788573228.355449:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573228.355450:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573228.355454:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355458:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573228.355459:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573228.355461:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573228.355462:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810083968, offset 0 00000400:00000200:0.0:1788573228.355464:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355467:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355469:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573228.355470:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573228.355472:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573228.355473:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573228.355475:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355477:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540770:540770:256:4294967295] 192.168.202.148@tcp LPNI seq info [540735:540735:8:4294967295] 00000400:00000200:0.0:1788573228.355480:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355483:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573228.355486:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355488:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573228.355493:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573228.355495:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573228.355502:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355504:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573228.355506:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573228.355508:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573228.355510:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573228.355511:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573228.355513:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573228.355515:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355520:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355522:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355526:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355527:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573228.355529:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573228.355531:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573228.355533:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573228.355534:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308367521 00000100:00000001:0.0:1788573228.355536:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355537:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308367521 00000100:00000001:0.0:1788573228.355539:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355540:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573228.355541:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355543:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573228.355545:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573228/real 1788573228] req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573228.355549:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573228.355553:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573228.355554:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573228.355555:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573228.355557:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573228.355558:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573228.355560:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573228.355566:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355570:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573228.355571:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355572:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355573:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573228.355575:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355579:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573228.355581:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355585:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573228.355587:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355588:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355588:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355589:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573228.355591:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573228.355592:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573228.355594:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573228.355596:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573228.355598:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573228.355610:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810083968:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573228.355612:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355614:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355616:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810083712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355620:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573228.355621:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355622:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573228.355624:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573228.355625:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573228.355626:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573228.355628:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573228.355629:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573228.355631:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573228.355632:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573228.355633:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573228.355650:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573228.355651:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355652:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355654:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810083840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355658:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573228.355659:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355660:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573228.355661:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573228.355662:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573228.355663:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573228.355664:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573228.355665:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573228.355666:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573228.355667:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573228.355668:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573228.355673:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573228.355674:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355675:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355677:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810083968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355681:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573228.355682:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355683:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573228.355684:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573228.355685:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573228.355686:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573228.355687:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573228.355688:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573228.355689:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573228.355689:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573228.355690:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573228.355691:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573228.355692:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355693:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355696:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573228.355697:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355698:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573228.355699:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573228.355700:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573228.355703:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355707:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.355708:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355709:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573228.355711:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573228.355712:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.355714:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573228.355715:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573228.355717:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573228.355718:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573228.355722:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355724:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573228.355725:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573228.355727:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573228.355729:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573228.355731:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573228.355734:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084096, portal 4 00000100:00000001:0.0:1788573228.355735:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573228.355736:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573228.355740:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355745:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573228.355746:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573228.355747:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573228.355749:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810084096, offset 0 00000400:00000200:0.0:1788573228.355751:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355754:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355756:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573228.355758:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573228.355760:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573228.355761:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573228.355763:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573228.355765:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540771:540771:256:4294967295] 192.168.202.148@tcp LPNI seq info [540736:540736:8:4294967295] 00000400:00000200:0.0:1788573228.355768:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573228.355772:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573228.355774:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355776:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573228.355779:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573228.355780:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573228.355788:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573228.355789:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0600. 00000400:00000200:0.0:1788573228.355794:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573228.355796:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573228.355798:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31e58 00000400:00000010:0.0:1788573228.355800:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31e58. 00000100:00000001:0.0:1788573228.355801:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573228.355803:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355809:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355811:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355815:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355816:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573228.355818:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573228.355821:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573228.355823:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573228.355824:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308367531 00000100:00000001:0.0:1788573228.355826:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355828:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308367531 00000100:00000001:0.0:1788573228.355830:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308371166 00000100:00000001:0.0:1788573228.355832:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573228.355834:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355836:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573228.355840:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573228/real 1788573228] req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573228.355848:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573228.355849:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573228.355851:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573228.355854:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573228.355856:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573228.355858:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573228.355869:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573228.355879:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355885:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573228.355887:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355888:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355889:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573228.355892:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355896:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573228.355899:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355903:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573228.355905:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355905:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573228.355906:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355907:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573228.355909:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573228.355910:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573228.355912:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573228.355913:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573228.355916:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573228.355924:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573228.355928:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355942:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573228.355946:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810084096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573228.355958:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573228.355960:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573228.355962:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573228.355965:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573228.355966:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573228.355968:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573228.355970:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573228.355971:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573228.355974:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573228.355975:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573228.355976:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573228.355978:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573228.355978:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355980:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.355982:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573228.355983:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573228.355985:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573228.355986:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573232.193742:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308371166, at 4308371200 00000100:00000010:0.0:1788573232.193758:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573233.345783:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573233.345788:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573233.345792:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573233.474113:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573233.474124:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573233.474130:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573233.474153:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.474204:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.474209:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.474218:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573233.474227:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573233.474240:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.474256:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810084224:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573233.474299:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573233.474309:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573233.474316:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.474327:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573233.474339:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573233.474344:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.474366:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573233.474374:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573233.474391:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573233.474406:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084224, portal 25 00000100:00000001:0.0:1788573233.474417:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573233.474422:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291926656 : -109714417624960 : ffff9c371f1c5880) 00000100:00000040:0.0:1788573233.474440:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.474479:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573233.474489:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573233.474493:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31e58. 00000100:00000200:0.0:1788573233.474500:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810084224, offset 0 00000400:00000200:0.0:1788573233.474517:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573233.474533:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.474551:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573233.474561:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573233.474571:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573233.474578:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573233.474584:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573233.474600:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540772:540772:256:4294967295] 192.168.202.148@tcp LPNI seq info [540737:540737:8:4294967295] 00000400:00000200:0.0:1788573233.474617:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.474697:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573233.474723:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573233.474791:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0600. 00000800:00000200:0.0:1788573233.474806:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573233.474871:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.474883:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573233.474887:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308376287 00000100:00000010:0.0:1788573233.474901:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573233.474912:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573233.474925:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308376287 00000100:00000010:0.0:1788573233.474929:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626c00. 00000100:00000001:0.0:1788573233.474933:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573233.474940:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573233.474944:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.474948:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.474954:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573233.474965:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573233.474970:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573233.474980:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573233.474986:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573233.474988:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573233.474999:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475036:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.475039:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475053:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573233.475056:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573233.475065:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.475071:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084352:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573233.475074:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573233.475075:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573233.475077:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475078:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573233.475082:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573233.475083:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.475086:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573233.475089:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b58. 00000400:00000010:0.0:1788573233.475098:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788573233.475101:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084352, portal 10 00000100:00000001:0.0:1788573233.475103:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573233.475104:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788573233.475110:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475115:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573233.475116:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573233.475118:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31110. 00000100:00000200:0.0:1788573233.475120:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810084352, offset 0 00000400:00000200:0.0:1788573233.475122:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475125:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475129:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573233.475131:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573233.475132:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573233.475134:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573233.475136:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475139:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540773:540773:255:4294967295] 192.168.202.148@tcp LPNI seq info [540738:540738:7:4294967295] 00000400:00000200:0.0:1788573233.475142:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475146:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573233.475149:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573233.475151:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788573233.475155:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573233.475160:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573233.475163:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573233.475165:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573233.475167:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308372505 00000100:00000001:0.0:1788573233.475169:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475170:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308372505 00000100:00000001:0.0:1788573233.475172:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475176:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573233.475178:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475179:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573233.475182:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475188:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.475189:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475191:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573233.475192:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573233.475193:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.475196:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573233.475199:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573233.475202:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573233.475203:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475205:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573233.475208:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573233.475210:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.475213:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573233.475216:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8840. 00000400:00000010:0.0:1788573233.475220:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573233.475224:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084480, portal 10 00000100:00000001:0.0:1788573233.475226:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573233.475227:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788573233.475234:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475242:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573233.475244:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573233.475247:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31990. 00000100:00000200:0.0:1788573233.475249:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810084480, offset 0 00000400:00000200:0.0:1788573233.475253:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475256:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475259:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573233.475265:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573233.475268:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573233.475269:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573233.475271:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475273:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540774:540774:254:4294967295] 192.168.202.148@tcp LPNI seq info [540739:540739:6:4294967295] 00000400:00000200:0.0:1788573233.475276:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475280:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573233.475283:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573233.475286:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573233.475288:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573233.475293:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573233.475297:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573233.475300:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573233.475302:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308372509 00000100:00000001:0.0:1788573233.475304:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475307:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308372509 00000100:00000001:0.0:1788573233.475309:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475311:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573233.475314:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475316:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573233.475321:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475329:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.475331:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475333:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573233.475335:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573233.475337:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.475340:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084608:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573233.475344:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573233.475351:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573233.475352:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475354:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573233.475356:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573233.475357:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.475360:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573233.475362:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c87e8. 00000400:00000010:0.0:1788573233.475365:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573233.475367:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084608, portal 4 00000100:00000001:0.0:1788573233.475369:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573233.475369:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000040:0.0:1788573233.475374:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475379:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573233.475381:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573233.475383:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31d48. 00000100:00000200:0.0:1788573233.475385:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810084608, offset 0 00000400:00000200:0.0:1788573233.475390:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475394:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475398:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573233.475401:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573233.475404:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573233.475406:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573233.475409:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475412:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540775:540775:253:4294967295] 192.168.202.148@tcp LPNI seq info [540740:540740:5:4294967295] 00000400:00000200:0.0:1788573233.475416:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475423:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573233.475428:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573233.475432:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788573233.475435:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573233.475440:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573233.475444:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573233.475453:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573233.475455:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308372641 00000100:00000001:0.0:1788573233.475458:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475461:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308372641 00000100:00000001:0.0:1788573233.475463:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475465:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573233.475468:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475470:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475473:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573233.475478:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573233.475479:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573233.475481:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573233.475483:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573233.475485:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573233.475490:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475497:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.475498:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475501:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573233.475503:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573233.475505:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.475508:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573233.475511:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573233.475513:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573233.475515:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573233.475517:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573233.475520:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573233.475522:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573233.475526:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573233.475528:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573233.475532:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573233.475535:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084736, portal 4 00000100:00000001:0.0:1788573233.475543:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573233.475544:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788573233.475552:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573233.475559:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573233.475562:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573233.475565:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31660. 00000100:00000200:0.0:1788573233.475568:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810084736, offset 0 00000400:00000200:0.0:1788573233.475572:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475576:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475581:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573233.475584:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573233.475586:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573233.475588:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573233.475590:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573233.475593:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540776:540776:252:4294967295] 192.168.202.148@tcp LPNI seq info [540741:540741:4:4294967295] 00000400:00000200:0.0:1788573233.475597:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573233.475604:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573233.475608:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573233.475613:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573233.475617:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573233.475624:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573233.475627:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573233.475631:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573233.475633:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308372650 00000100:00000001:0.0:1788573233.475663:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475666:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308372651 00000100:00000001:0.0:1788573233.475668:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308376287 00000100:00000001:0.0:1788573233.475669:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573233.475681:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475684:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475687:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573233.475690:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573233.475692:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573233.475694:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573233.475695:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573233.475697:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.475699:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573236.546166:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573236.546172:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573236.546178:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573236.546191:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573233/real 1788573236] req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573236.546207:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573236.546210:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573236.546213:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8b58. 00000400:00000200:0.0:1788573236.546218:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31aa0 00000400:00000010:0.0:1788573236.546220:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31aa0. 00000100:00000001:0.0:1788573236.546223:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573236.546230:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573236.546247:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546268:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573236.546271:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546292:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546296:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573236.546306:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546336:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573236.546346:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546392:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573236.546398:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546402:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546404:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546409:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573236.546414:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573236.546418:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573236.546426:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573236.546432:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573236.546442:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573236.546463:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084352:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573236.546475:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573236.546481:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573233/real 1788573236] req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573236.546490:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573236.546492:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573236.546494:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8840. 00000400:00000200:0.0:1788573236.546498:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed316e8 00000400:00000010:0.0:1788573236.546500:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed316e8. 00000100:00000001:0.0:1788573236.546502:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573236.546506:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573236.546513:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546519:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573236.546527:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546529:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546530:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573236.546549:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546555:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573236.546559:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546565:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573236.546567:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546569:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546570:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546572:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573236.546574:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573236.546581:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573236.546585:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573236.546587:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573236.546591:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573236.546598:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573236.546602:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573236.546606:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573233/real 1788573236] req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573236.546614:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573236.546616:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573236.546618:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c87e8. 00000400:00000200:0.0:1788573236.546621:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573236.546623:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573236.546685:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573236.546692:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573236.546703:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546718:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573236.546720:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546723:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546724:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573236.546730:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546738:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573236.546742:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546749:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573236.546752:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546754:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546755:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546757:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573236.546760:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573236.546762:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573236.546767:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573236.546770:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573236.546773:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573236.546796:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084608:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573236.546853:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573236.546866:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573233/real 1788573236] req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573236.546877:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573236.546879:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573236.546881:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573236.546885:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573236.546887:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573236.546891:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573236.546896:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573236.546906:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546920:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573236.546922:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546924:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546925:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573236.546929:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546936:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573236.546940:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.546948:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573236.546951:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546953:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.546954:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.546956:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573236.546958:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573236.546960:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573236.546964:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573236.546978:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573236.546990:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573236.546999:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810084224:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573236.547004:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573236.547007:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573236.547011:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810084352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547019:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573236.547023:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573236.547027:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573236.547031:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573236.547034:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573236.547036:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573236.547039:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573236.547041:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788573236.547045:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573236.547047:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573236.547049:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788573236.547053:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573236.547055:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547056:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573236.547060:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810084480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547066:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573236.547068:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573236.547069:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573236.547072:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573236.547073:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573236.547074:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573236.547076:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573236.547078:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e800. 02000000:00000001:0.0:1788573236.547080:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573236.547087:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573236.547089:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788573236.547091:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573236.547092:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547093:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573236.547096:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440810084608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547101:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573236.547101:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573236.547103:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573236.547105:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573236.547105:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573236.547106:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573236.547107:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573236.547113:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233cc00. 02000000:00000001:0.0:1788573236.547115:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573236.547116:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573236.547117:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788573236.547118:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573236.547119:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547120:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573236.547123:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440810084224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547129:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573236.547130:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573236.547132:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573236.547134:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573236.547136:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573236.547137:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573236.547139:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573236.547140:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788573236.547142:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573236.547143:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573236.547144:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788573236.547146:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573236.547147:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547157:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573236.547160:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573236.547161:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573236.547163:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573236.547168:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573233/real 1788573236] req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573236.547178:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573236.547180:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573236.547181:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573236.547183:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573236.547184:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573236.547186:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573236.547189:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573236.547194:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547199:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573236.547200:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547201:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.547202:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573236.547204:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547208:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573236.547211:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547220:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573236.547222:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547222:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573236.547223:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547225:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573236.547226:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573236.547231:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573236.547234:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573236.547236:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573236.547240:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573236.547248:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573236.547252:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547254:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573236.547257:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440810084736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573236.547263:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573236.547265:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573236.547267:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573236.547270:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573236.547271:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573236.547272:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573236.547274:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573236.547277:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788573236.547279:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573236.547280:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573236.547282:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788573236.547284:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573236.547286:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547288:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.547294:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573236.547296:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573236.547299:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573236.547300:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573237.313713:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308376287, at 4308376320 00000100:00000010:0.0:1788573237.313726:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573238.465866:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573238.465887:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573238.465891:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573238.594912:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573238.594916:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573238.594923:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573238.594930:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.594938:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.594940:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.594944:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573238.594947:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573238.594949:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.594953:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810084864:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573238.594957:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573238.594962:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573238.594964:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.594967:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573238.594972:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573238.594974:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.594980:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573238.594984:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573238.594991:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573238.594998:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084864, portal 25 00000100:00000001:0.0:1788573238.595001:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573238.595003:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788573238.595008:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595017:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573238.595020:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573238.595022:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573238.595025:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810084864, offset 0 00000400:00000200:0.0:1788573238.595180:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595187:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.595192:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573238.595202:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573238.595205:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573238.595208:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573238.595210:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595213:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540777:540777:256:4294967295] 192.168.202.148@tcp LPNI seq info [540742:540742:8:4294967295] 00000400:00000200:0.0:1788573238.595217:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.595224:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573238.595229:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.595234:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573238.595242:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573238.595245:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573238.595260:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.595263:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573238.595268:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573238.595273:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573238.595280:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573238.595283:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573238.595287:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.595292:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595302:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.595306:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595314:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.595316:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.595323:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.595328:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573238.595331:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308381407 00000100:00000010:0.0:1788573238.595336:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573238.595343:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573238.595352:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308381407 00000100:00000010:0.0:1788573238.595355:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626de0. 00000100:00000001:0.0:1788573238.595357:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573238.595359:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573238.595362:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.595365:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573238.595370:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573238/real 1788573238] req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573238.595382:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573238.595384:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573238.595386:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573238.595389:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573238.595391:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573238.595394:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573238.595398:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573238.595407:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595414:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573238.595416:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595418:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.595420:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573238.595425:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595433:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573238.595438:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595446:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573238.595449:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595450:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.595459:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595461:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573238.595463:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573238.595467:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573238.595471:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573238.595487:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573238.595493:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573238.595503:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810084864:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573238.595508:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573238.595512:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595518:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.595520:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.595523:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573238.595524:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573238.595526:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.595530:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084992:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573238.595533:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573238.595535:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573238.595537:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.595539:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573238.595544:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573238.595545:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.595551:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573238.595554:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573238.595558:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573238.595562:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810084992, portal 10 00000100:00000001:0.0:1788573238.595564:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573238.595566:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573238.595579:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595586:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573238.595589:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573238.595591:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573238.595595:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810084992, offset 0 00000400:00000200:0.0:1788573238.595601:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595605:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.595609:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573238.595613:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573238.595616:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573238.595619:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573238.595622:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595626:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540778:540778:256:4294967295] 192.168.202.148@tcp LPNI seq info [540743:540743:8:4294967295] 00000400:00000200:0.0:1788573238.595664:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.595672:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573238.595678:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.595682:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573238.595687:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573238.595689:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573238.595699:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.595701:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573238.595704:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573238.595706:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573238.595708:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573238.595710:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573238.595712:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.595715:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595721:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.595730:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595734:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.595749:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.595755:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573238.595758:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573238.595762:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573238.595764:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308377626 00000100:00000001:0.0:1788573238.595767:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.595770:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308377626 00000100:00000001:0.0:1788573238.595772:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.595774:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573238.595776:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.595779:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573238.595783:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573238/real 1788573238] req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573238.595792:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573238.595794:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573238.595796:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573238.595799:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573238.595801:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573238.595803:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573238.595807:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573238.595815:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595828:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573238.595829:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595831:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.595832:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573238.595847:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595854:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573238.595858:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595864:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573238.595867:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595869:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.595870:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.595871:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573238.595873:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573238.595876:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573238.595879:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573238.595881:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573238.595885:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573238.595893:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810084992:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573238.595897:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573238.595900:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595906:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.595908:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.595910:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573238.595912:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573238.595914:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.595917:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085120:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573238.595920:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573238.595927:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573238.595928:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.595930:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573238.595933:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573238.595935:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.595939:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573238.595942:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573238.595946:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573238.595949:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810085120, portal 10 00000100:00000001:0.0:1788573238.595951:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573238.595952:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573238.595959:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.595965:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573238.595968:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573238.595970:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573238.595973:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810085120, offset 0 00000400:00000200:0.0:1788573238.595977:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595981:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.595984:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573238.595987:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573238.595990:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573238.595992:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573238.595995:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573238.595998:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540779:540779:256:4294967295] 192.168.202.148@tcp LPNI seq info [540744:540744:8:4294967295] 00000400:00000200:0.0:1788573238.596002:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.596014:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573238.596020:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596024:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573238.596028:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573238.596046:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573238.596054:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596060:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573238.596063:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573238.596065:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573238.596068:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573238.596069:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573238.596072:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.596076:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596085:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596088:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596095:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596097:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.596100:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573238.596104:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573238.596108:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573238.596110:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308377630 00000100:00000001:0.0:1788573238.596113:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596116:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308377630 00000100:00000001:0.0:1788573238.596118:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596120:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573238.596122:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596124:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573238.596128:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573238/real 1788573238] req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573238.596135:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573238.596137:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573238.596138:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573238.596141:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573238.596142:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573238.596151:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573238.596154:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573238.596163:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596169:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573238.596171:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596173:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596174:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573238.596177:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596184:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573238.596187:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596193:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573238.596196:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596197:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596198:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596200:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573238.596202:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573238.596205:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573238.596208:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573238.596210:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573238.596214:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573238.596223:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085120:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573238.596226:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573238.596236:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596243:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.596245:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.596247:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573238.596249:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573238.596251:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.596254:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085248:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573238.596258:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573238.596260:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573238.596262:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.596263:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573238.596266:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573238.596268:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.596273:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573238.596276:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573238.596280:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573238.596283:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810085248, portal 4 00000100:00000001:0.0:1788573238.596285:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573238.596286:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573238.596292:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596298:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573238.596300:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573238.596302:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573238.596304:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810085248, offset 0 00000400:00000200:0.0:1788573238.596308:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573238.596312:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.596315:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573238.596318:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573238.596320:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573238.596323:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573238.596325:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573238.596333:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540780:540780:256:4294967295] 192.168.202.148@tcp LPNI seq info [540745:540745:8:4294967295] 00000400:00000200:0.0:1788573238.596338:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.596345:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573238.596350:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596354:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573238.596358:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573238.596361:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573238.596373:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596375:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573238.596379:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573238.596382:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573238.596386:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573238.596388:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573238.596391:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.596395:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596404:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596407:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596420:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596421:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.596425:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573238.596429:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573238.596432:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573238.596434:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308377762 00000100:00000001:0.0:1788573238.596437:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596439:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308377762 00000100:00000001:0.0:1788573238.596442:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596444:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573238.596454:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596456:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573238.596461:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573238/real 1788573238] req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573238.596470:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573238.596472:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573238.596473:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573238.596476:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573238.596478:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573238.596481:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573238.596485:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573238.596493:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596499:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573238.596501:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596503:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596504:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573238.596508:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596520:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573238.596524:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596531:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573238.596534:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596535:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596537:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596539:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573238.596541:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573238.596544:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573238.596554:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573238.596557:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573238.596560:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573238.596566:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085248:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573238.596569:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573238.596571:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596577:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.596578:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.596581:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573238.596583:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573238.596585:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.596588:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085376:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573238.596591:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573238.596593:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573238.596595:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573238.596597:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573238.596600:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573238.596602:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573238.596607:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573238.596609:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573238.596613:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573238.596617:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810085376, portal 4 00000100:00000001:0.0:1788573238.596619:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573238.596620:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573238.596627:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596634:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573238.596668:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573238.596671:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573238.596674:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810085376, offset 0 00000400:00000200:0.0:1788573238.596679:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573238.596684:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.596688:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573238.596691:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573238.596693:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573238.596696:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573238.596698:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573238.596701:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540781:540781:256:4294967295] 192.168.202.148@tcp LPNI seq info [540746:540746:8:4294967295] 00000400:00000200:0.0:1788573238.596706:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573238.596712:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573238.596717:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596721:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573238.596726:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573238.596729:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573238.596762:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573238.596764:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573238.596768:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573238.596771:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573238.596775:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573238.596776:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573238.596778:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.596781:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596788:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596791:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596797:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596799:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.596809:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573238.596812:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573238.596816:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573238.596818:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308377772 00000100:00000001:0.0:1788573238.596821:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596823:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308377772 00000100:00000001:0.0:1788573238.596825:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308381407 00000100:00000001:0.0:1788573238.596828:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573238.596830:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.596832:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573238.596837:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573238/real 1788573238] req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573238.596845:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573238.596847:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573238.596849:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573238.596851:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573238.596852:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573238.596854:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573238.596856:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573238.596862:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596866:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573238.596867:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596868:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596870:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573238.596872:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596876:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573238.596879:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596888:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573238.596889:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596890:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573238.596891:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596892:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573238.596894:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573238.596895:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573238.596897:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573238.596898:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573238.596901:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573238.596906:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810085376:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573238.596909:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596910:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596913:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810084864/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596917:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573238.596919:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573238.596921:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573238.596924:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573238.596926:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573238.596927:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573238.596929:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573238.596931:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573238.596933:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573238.596935:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573238.596936:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573238.596939:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573238.596940:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596943:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596946:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810084992/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596950:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573238.596951:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573238.596952:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573238.596954:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573238.596954:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573238.596955:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573238.596956:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573238.596957:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573238.596958:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573238.596959:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573238.596960:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573238.596961:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573238.596962:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596963:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596965:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810085120/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596969:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573238.596970:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573238.596972:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573238.596974:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573238.596975:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573238.596976:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573238.596977:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573238.596977:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573238.596979:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573238.596979:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573238.596980:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573238.596981:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573238.596982:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.596983:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.596985:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810085248/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.596998:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573238.596999:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573238.597000:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788573238.597001:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573238.597002:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573238.597002:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573238.597003:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573238.597004:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573238.597006:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573238.597006:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573238.597007:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573238.597008:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573238.597009:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.597010:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.597012:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810085376/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.597016:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573238.597017:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573238.597017:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788573238.597019:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573238.597019:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573238.597020:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573238.597021:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573238.597022:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788573238.597025:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573238.597026:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573238.597027:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573238.597029:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573238.597031:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.597033:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.597040:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573238.597042:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.597044:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573238.597046:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573242.433742:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308381407, at 4308381440 00000100:00000010:0.0:1788573242.433772:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573243.585911:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573243.585915:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573243.585919:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573243.713774:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573243.713781:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573243.713784:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573243.713788:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573243.713791:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573243.713799:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13710 00000400:00000200:0.0:1788573243.713809:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573243.713818:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573243.713822:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573243.713837:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.713841:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.713846:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.713849:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573243.713853:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573243.713854:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573243.713856:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573243.713858:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573243.713862:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573243.713865:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573243.713868:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.713871:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.713873:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.713875:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573243.713880:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573243.713884:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573243.713887:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573243.713900:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573243.713905:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573243.713909:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.713915:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573243.713918:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573243.713924:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573243.713929:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573243.713934:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573243.713936:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.713940:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573243.713944:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573243.713950:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573243.713958:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573243.713965:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.713973:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810085504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573243.713982:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573243.713985:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.713989:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810085504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573243.714009:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573243.714012:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573243.714014:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573243.714016:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573243.714017:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573243.714019:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573243.714021:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13710 00000400:00000200:0.0:1788573243.714024:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573243.714026:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573243.714028:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573243.714031:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714032:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714039:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714040:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573243.714044:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573243.714046:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573243.714049:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573243.714053:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573243.714057:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573243.714059:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573243.714064:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714068:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714072:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714076:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573243.714080:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573243.714086:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714088:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573243.714090:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573243.714094:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573243.714097:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714100:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573243.714104:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573243.714106:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714107:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573243.714114:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573243.714117:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573243.714121:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573243.714123:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714127:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573243.714129:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573243.714132:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573243.714137:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573243.714140:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.714149:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810085632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573243.714162:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573243.714166:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.714170:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810085632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573243.714177:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573243.714180:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573243.714185:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573243.714186:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573243.714187:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573243.714190:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573243.714194:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13710 00000400:00000200:0.0:1788573243.714199:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573243.714202:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573243.714207:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573243.714209:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714210:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714212:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714217:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573243.714220:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573243.714221:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573243.714222:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573243.714223:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573243.714225:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573243.714231:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573243.714233:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714237:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714241:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714244:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573243.714251:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573243.714255:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714259:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573243.714260:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573243.714262:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573243.714264:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714265:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573243.714267:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573243.714272:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714275:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573243.714279:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573243.714283:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573243.714286:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573243.714290:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714292:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573243.714294:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573243.714297:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573243.714302:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573243.714305:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.714313:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810085760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573243.714321:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573243.714324:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.714329:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810085760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573243.714336:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573243.714339:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573243.714343:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573243.714344:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573243.714347:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573243.714353:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573243.714363:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13710 00000400:00000200:0.0:1788573243.714399:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573243.714404:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573243.714407:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573243.714412:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714413:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714418:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714423:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573243.714425:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573243.714427:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573243.714429:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573243.714433:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573243.714436:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573243.714439:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573243.714444:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714447:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714450:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714454:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573243.714457:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573243.714462:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714467:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573243.714469:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573243.714472:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573243.714475:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714477:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573243.714480:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573243.714483:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714486:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573243.714490:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573243.714494:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573243.714497:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573243.714514:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714516:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573243.714519:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573243.714521:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573243.714526:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573243.714532:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.714539:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810085888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573243.714560:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573243.714562:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.714565:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810085888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573243.714577:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573243.714581:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573243.714585:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573243.714586:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573243.714590:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573243.714593:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573243.714598:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13710 00000400:00000200:0.0:1788573243.714604:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573243.714608:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573243.714611:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573243.714614:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714619:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714622:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714623:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573243.714629:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573243.714630:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573243.714633:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573243.714687:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573243.714690:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573243.714698:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573243.714706:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714711:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573243.714713:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714715:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573243.714717:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573243.714719:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573243.714723:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573243.714727:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573243.714730:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573243.714734:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714738:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573243.714740:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573243.714743:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714745:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573243.714748:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573243.714752:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573243.714756:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573243.714758:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.714762:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573243.714763:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573243.714766:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573243.714770:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573243.714775:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.714778:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810086016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573243.714784:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573243.714787:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.714797:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810086016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573243.714817:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573243.714825:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13720) 00000100:00080000:0.0:1788573245.058427:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573245.058432:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573245.058434:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573245.058436:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573245.058439:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573245.058442:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13715 00000400:00000200:0.0:1788573245.058449:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573245.058454:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573245.058456:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573245.058459:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058461:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058463:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058465:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573245.058467:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573245.058468:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573245.058470:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573245.058471:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573245.058473:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573245.058475:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573245.058476:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058478:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058479:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058481:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573245.058488:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573245.058491:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058492:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573245.058494:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573245.058496:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573245.058498:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058507:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573245.058509:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573245.058514:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573245.058517:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573245.058521:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573245.058523:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058525:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573245.058527:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573245.058529:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573245.058534:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573245.058538:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.058545:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810086272/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573245.058554:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573245.058557:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.058562:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810086272/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573245.058587:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573245.058591:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573245.058595:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573245.058596:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573245.058597:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573245.058599:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573245.058601:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13715 00000400:00000200:0.0:1788573245.058605:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573245.058607:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573245.058609:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573245.058612:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058629:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058632:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058673:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573245.058681:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573245.058682:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573245.058683:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573245.058686:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573245.058688:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573245.058689:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573245.058692:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058694:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058696:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058697:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573245.058699:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573245.058703:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058705:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573245.058707:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573245.058712:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573245.058715:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058720:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573245.058725:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573245.058730:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058732:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573245.058735:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573245.058736:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573245.058738:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573245.058739:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058741:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573245.058742:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573245.058744:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573245.058747:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573245.058750:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.058757:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810086400/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573245.058769:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573245.058777:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.058784:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810086400/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573245.058797:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573245.058810:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573245.058817:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573245.058820:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573245.058824:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573245.058830:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573245.058835:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13715 00000400:00000200:0.0:1788573245.058843:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573245.058849:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573245.058854:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573245.058861:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058867:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058869:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058873:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573245.058878:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573245.058879:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573245.058882:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573245.058886:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573245.058891:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573245.058899:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573245.058907:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058913:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.058918:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058924:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573245.058931:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573245.058939:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573245.058955:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573245.058961:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573245.058967:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573245.058972:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058977:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573245.058982:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573245.058987:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.058990:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573245.058995:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573245.058999:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573245.059004:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573245.059006:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059008:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573245.059009:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573245.059011:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573245.059017:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573245.059022:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.059030:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810086528/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573245.059040:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573245.059042:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.059048:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810086528/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573245.059058:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573245.059063:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573245.059066:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573245.059069:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573245.059070:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573245.059074:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573245.059078:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13715 00000400:00000200:0.0:1788573245.059085:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573245.059095:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573245.059100:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573245.059103:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059107:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.059109:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059110:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573245.059114:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573245.059117:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573245.059119:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573245.059121:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573245.059124:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573245.059129:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573245.059136:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059141:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.059147:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059152:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573245.059158:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573245.059166:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059174:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573245.059177:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573245.059182:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573245.059188:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059194:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573245.059198:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573245.059206:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059211:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573245.059219:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573245.059223:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573245.059228:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573245.059230:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059241:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573245.059247:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573245.059252:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573245.059255:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573245.059261:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.059270:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810086656/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573245.059278:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573245.059281:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.059286:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810086656/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573245.059300:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573245.059305:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573245.059312:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573245.059313:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573245.059315:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573245.059320:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573245.059326:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13715 00000400:00000200:0.0:1788573245.059332:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573245.059337:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573245.059342:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573245.059345:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059348:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.059352:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059357:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573245.059361:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573245.059363:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573245.059365:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573245.059367:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573245.059368:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573245.059369:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573245.059375:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059380:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573245.059383:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059388:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573245.059390:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573245.059391:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573245.059393:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573245.059394:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573245.059397:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573245.059399:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059400:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573245.059403:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573245.059407:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059410:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573245.059415:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573245.059419:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573245.059423:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573245.059425:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573245.059428:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573245.059429:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573245.059432:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573245.059436:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573245.059440:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.059447:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810086784/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573245.059466:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573245.059470:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.059474:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810086784/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573245.059484:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573245.059486:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13721) 00000020:00000001:0.0:1788573248.705780:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573248.705784:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573248.705787:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573248.897752:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308387871, at 4308387904 00000100:00000010:0.0:1788573248.897771:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00080000:0.0:1788573250.369787:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573250.369799:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573250.369802:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573250.369806:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573250.369810:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573250.369816:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13716 00000400:00000200:0.0:1788573250.369825:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573250.369831:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573250.369833:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573250.369836:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.369839:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.369841:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.369843:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573250.369846:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573250.369848:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573250.369850:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573250.369854:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573250.369855:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573250.369857:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573250.369859:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.369861:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.369862:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.369863:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573250.369866:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573250.369874:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573250.369894:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573250.369896:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573250.369899:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573250.369901:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.369903:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573250.369905:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573250.369910:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573250.369913:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573250.369916:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573250.369918:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.369920:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573250.369922:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573250.369926:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573250.369932:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573250.369937:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.369945:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810086912/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573250.369955:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573250.369960:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.369965:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810086912/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573250.369986:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573250.369990:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573250.369994:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573250.369996:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573250.369998:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573250.370002:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573250.370006:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13716 00000400:00000200:0.0:1788573250.370015:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573250.370023:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573250.370029:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573250.370036:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370045:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370050:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370054:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573250.370060:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573250.370062:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573250.370065:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573250.370068:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573250.370071:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573250.370074:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573250.370078:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370081:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370084:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370087:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573250.370089:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573250.370095:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370098:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573250.370102:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573250.370107:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573250.370111:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370112:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573250.370116:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573250.370121:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370125:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573250.370131:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573250.370136:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573250.370141:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573250.370144:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370147:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573250.370148:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573250.370150:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573250.370154:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573250.370160:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.370175:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810087040/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573250.370186:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573250.370188:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.370195:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810087040/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573250.370210:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573250.370215:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573250.370221:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573250.370224:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573250.370227:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573250.370233:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573250.370237:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13716 00000400:00000200:0.0:1788573250.370253:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573250.370259:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573250.370261:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573250.370263:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370268:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370270:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370271:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573250.370274:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573250.370276:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573250.370278:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573250.370281:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573250.370284:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573250.370286:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573250.370289:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370293:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370296:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370303:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573250.370307:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573250.370312:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370315:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573250.370319:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573250.370322:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573250.370324:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370326:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573250.370329:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573250.370332:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370333:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573250.370335:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573250.370339:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573250.370342:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573250.370345:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370350:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573250.370351:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573250.370353:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573250.370359:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573250.370363:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.370371:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810087168/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573250.370397:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573250.370399:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.370406:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810087168/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573250.370428:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573250.370432:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573250.370434:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573250.370435:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573250.370436:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573250.370446:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573250.370451:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13716 00000400:00000200:0.0:1788573250.370457:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573250.370461:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573250.370464:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573250.370469:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370471:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370472:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370473:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573250.370475:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573250.370476:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573250.370477:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573250.370479:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573250.370480:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573250.370481:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573250.370484:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370486:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370490:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370493:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573250.370496:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573250.370499:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370505:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573250.370511:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573250.370542:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573250.370549:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370555:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573250.370560:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573250.370566:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370573:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573250.370582:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573250.370595:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573250.370602:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573250.370606:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370612:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573250.370614:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573250.370618:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573250.370624:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573250.370630:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.370669:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810087296/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573250.370690:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573250.370694:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.370700:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810087296/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573250.370711:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573250.370714:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573250.370719:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573250.370720:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573250.370721:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573250.370723:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573250.370725:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13716 00000400:00000200:0.0:1788573250.370731:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573250.370737:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573250.370742:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573250.370748:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370753:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370758:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370763:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573250.370766:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573250.370767:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573250.370776:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573250.370779:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573250.370784:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573250.370789:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573250.370795:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370797:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573250.370802:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370806:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573250.370810:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573250.370815:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573250.370818:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573250.370822:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573250.370825:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573250.370830:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370835:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573250.370837:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573250.370842:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370845:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573250.370848:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573250.370852:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573250.370855:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573250.370856:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.370858:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573250.370862:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573250.370865:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573250.370871:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573250.370876:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.370885:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810087424/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573250.370897:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573250.370901:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.370909:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810087424/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573250.370927:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573250.370931:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13727) 00000020:00000001:0.0:1788573252.801817:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573252.801821:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573252.801824:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573253.825816:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573253.825820:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573253.825823:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573254.209728:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308393182, at 4308393216 00000100:00000010:0.0:1788573254.209749:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00080000:0.0:1788573255.489761:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573255.489801:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573255.489804:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573255.489809:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573255.489813:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573255.489820:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13722 00000400:00000200:0.0:1788573255.489829:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573255.489836:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573255.489839:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573255.489843:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.489846:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.489848:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.489851:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573255.489855:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573255.489857:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573255.489860:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573255.489863:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573255.489866:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573255.489869:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573255.489872:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.489890:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.489892:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.489895:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573255.489898:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573255.489901:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573255.489904:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573255.489907:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573255.489910:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573255.489914:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.489916:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573255.489918:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573255.489925:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573255.489929:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573255.489933:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573255.489935:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.489938:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573255.489941:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573255.489945:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573255.489952:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573255.489960:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.489968:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810087552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573255.489979:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573255.489984:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.489988:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810087552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573255.490017:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573255.490042:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573255.490047:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573255.490049:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573255.490052:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573255.490056:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573255.490072:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13722 00000400:00000200:0.0:1788573255.490082:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573255.490092:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573255.490098:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573255.490105:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490113:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490119:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490125:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573255.490130:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573255.490134:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573255.490139:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573255.490145:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573255.490149:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573255.490153:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573255.490160:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490164:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490170:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490177:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573255.490180:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573255.490185:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490191:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573255.490193:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573255.490203:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573255.490209:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490216:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573255.490223:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573255.490230:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490235:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573255.490243:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573255.490248:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573255.490255:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573255.490265:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490271:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573255.490275:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573255.490281:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573255.490291:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573255.490299:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.490311:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810087680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573255.490321:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573255.490324:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.490329:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810087680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573255.490345:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573255.490349:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573255.490354:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573255.490358:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573255.490362:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573255.490367:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573255.490375:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13722 00000400:00000200:0.0:1788573255.490386:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573255.490404:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573255.490412:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573255.490431:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490435:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490437:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490439:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573255.490442:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573255.490443:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573255.490445:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573255.490447:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573255.490449:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573255.490460:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573255.490470:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490476:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490479:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490482:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573255.490488:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573255.490494:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490499:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573255.490505:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573255.490511:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573255.490516:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490521:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573255.490524:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573255.490530:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490534:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573255.490539:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573255.490544:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573255.490550:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573255.490554:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490558:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573255.490560:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573255.490565:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573255.490572:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573255.490578:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.490589:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810087808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573255.490606:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573255.490612:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.490623:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810087808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573255.490664:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573255.490675:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573255.490680:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573255.490685:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573255.490689:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573255.490696:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573255.490705:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13722 00000400:00000200:0.0:1788573255.490716:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573255.490726:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573255.490732:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573255.490740:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490747:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490754:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490761:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573255.490768:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573255.490772:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573255.490779:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573255.490785:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573255.490791:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573255.490798:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573255.490806:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490809:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.490812:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490815:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573255.490822:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573255.490830:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573255.490836:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573255.490840:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573255.490846:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573255.490851:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490857:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573255.490870:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573255.490877:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490883:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573255.490891:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573255.490898:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573255.490904:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573255.490909:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.490917:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573255.490920:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573255.490923:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573255.490930:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573255.490939:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.490951:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810087936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573255.490971:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573255.490976:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.490985:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810087936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573255.491009:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573255.491015:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573255.491024:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573255.491028:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573255.491032:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573255.491039:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573255.491046:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13722 00000400:00000200:0.0:1788573255.491059:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573255.491070:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573255.491075:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573255.491083:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.491089:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.491097:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.491112:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573255.491119:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573255.491123:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573255.491128:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573255.491132:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573255.491136:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573255.491141:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573255.491149:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.491153:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573255.491158:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573255.491162:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573255.491167:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573255.491172:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573255.491175:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573255.491178:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573255.491183:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573255.491189:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.491194:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573255.491198:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573255.491203:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.491207:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573255.491212:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573255.491217:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573255.491224:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573255.491227:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.491231:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573255.491235:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573255.491240:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573255.491246:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573255.491252:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.491264:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810088064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573255.491321:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573255.491326:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.491335:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810088064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573255.491350:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573255.491356:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13732) 00000020:00000001:0.0:1788573257.921801:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573257.921805:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573257.921808:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573258.945815:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573258.945819:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573258.945822:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573259.009965:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573259.329746:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308398302, at 4308398336 00000100:00000010:0.0:1788573259.329763:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00000001:0.0:1788573260.609975:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573260.609982:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573260.609988:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573260.609999:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610021:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.610023:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.610029:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573260.610033:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573260.610036:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.610040:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810088192:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573260.610047:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573260.610052:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573260.610055:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.610061:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573260.610069:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573260.610080:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.610087:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573260.610092:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573260.610141:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573260.610152:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810088192, portal 25 00000100:00000001:0.0:1788573260.610159:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573260.610164:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788573260.610178:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610194:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573260.610203:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573260.610209:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573260.610217:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810088192, offset 0 00000400:00000200:0.0:1788573260.610224:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573260.610236:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.610246:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573260.610254:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573260.610261:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573260.610268:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573260.610276:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573260.610285:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540802:540802:256:4294967295] 192.168.202.148@tcp LPNI seq info [540767:540767:8:4294967295] 00000400:00000200:0.0:1788573260.610297:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.610316:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573260.610340:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.610373:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573260.610387:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573260.610394:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573260.610417:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.610420:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573260.610425:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573260.610432:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573260.610446:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573260.610450:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573260.610456:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573260.610462:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610473:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.610477:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610487:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.610489:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573260.610494:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.610502:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573260.610508:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308403422 00000100:00000010:0.0:1788573260.610514:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573260.610523:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573260.610532:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308403422 00000100:00000010:0.0:1788573260.610536:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626d20. 00000100:00000001:0.0:1788573260.610541:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573260.610546:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573260.610550:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.610555:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573260.610564:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573260/real 1788573260] req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573260.610576:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573260.610580:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573260.610584:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573260.610588:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573260.610592:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573260.610597:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573260.610603:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573260.610621:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610657:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573260.610659:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610661:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.610662:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573260.610667:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610674:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573260.610681:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610693:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573260.610698:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610701:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.610704:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610708:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573260.610713:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573260.610718:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573260.610724:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573260.610733:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573260.610742:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573260.610766:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810088192:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573260.610777:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610780:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.610788:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440810088192/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610798:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573260.610802:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573260.610809:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573260.610812:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573260.610813:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573260.610815:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573260.610817:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573260.610819:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788573260.610828:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573260.610830:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573260.610831:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788573260.610834:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573260.610835:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610836:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.610842:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573260.610843:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.610845:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573260.610846:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573260.610847:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573260.610850:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610855:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.610856:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.610858:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573260.610859:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573260.610861:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.610863:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088320:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573260.610865:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573260.610866:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573260.610868:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.610869:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573260.610871:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573260.610872:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.610875:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573260.610877:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573260.610880:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573260.610882:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810088320, portal 10 00000100:00000001:0.0:1788573260.610887:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573260.610888:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291929344 : -109714417622272 : ffff9c371f1c6300) 00000100:00000040:0.0:1788573260.610893:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610897:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573260.610899:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573260.610900:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573260.610902:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810088320, offset 0 00000400:00000200:0.0:1788573260.610905:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573260.610908:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.610910:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573260.610913:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573260.610915:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573260.610916:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573260.610918:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573260.610920:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540803:540803:256:4294967295] 192.168.202.148@tcp LPNI seq info [540768:540768:8:4294967295] 00000400:00000200:0.0:1788573260.610923:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.610928:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573260.610931:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.610933:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573260.610936:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573260.610938:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573260.610948:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.610950:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573260.610953:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573260.610957:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573260.610960:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573260.610962:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573260.610965:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573260.610968:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.610989:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.610993:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611000:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611001:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573260.611004:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573260.611007:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573260.611009:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573260.611011:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308399641 00000100:00000001:0.0:1788573260.611013:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611014:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308399641 00000100:00000001:0.0:1788573260.611016:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611017:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573260.611019:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611020:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573260.611023:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573260/real 1788573260] req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573260.611028:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573260.611029:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573260.611030:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573260.611032:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573260.611033:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573260.611035:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573260.611037:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573260.611042:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611046:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573260.611051:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611053:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611054:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573260.611056:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611061:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573260.611064:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611068:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573260.611070:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611071:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611072:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611073:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573260.611074:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573260.611076:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573260.611078:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573260.611079:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573260.611081:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573260.611086:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088320:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573260.611089:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573260.611091:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611095:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.611096:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611098:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573260.611099:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573260.611100:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.611106:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088448:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573260.611108:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573260.611110:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573260.611111:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611112:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611114:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573260.611115:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.611118:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573260.611119:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573260.611121:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573260.611124:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810088448, portal 10 00000100:00000001:0.0:1788573260.611125:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573260.611126:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291933824 : -109714417617792 : ffff9c371f1c7480) 00000100:00000040:0.0:1788573260.611130:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611134:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573260.611136:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573260.611137:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573260.611138:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810088448, offset 0 00000400:00000200:0.0:1788573260.611141:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611143:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611146:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573260.611148:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573260.611149:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573260.611151:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573260.611152:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611155:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540804:540804:256:4294967295] 192.168.202.148@tcp LPNI seq info [540769:540769:8:4294967295] 00000400:00000200:0.0:1788573260.611157:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611163:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573260.611167:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611170:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573260.611179:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573260.611182:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573260.611193:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611195:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573260.611199:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573260.611203:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573260.611206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573260.611208:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573260.611211:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573260.611215:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611220:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611222:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611227:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611228:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573260.611231:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573260.611233:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573260.611235:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573260.611237:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308399645 00000100:00000001:0.0:1788573260.611238:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611240:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308399645 00000100:00000001:0.0:1788573260.611241:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611243:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573260.611244:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611245:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573260.611248:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573260/real 1788573260] req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573260.611252:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573260.611257:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573260.611259:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573260.611261:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573260.611263:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573260.611266:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573260.611269:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573260.611278:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611285:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573260.611287:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611288:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611290:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573260.611294:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611302:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573260.611307:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611312:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573260.611314:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611315:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611316:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611317:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573260.611318:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573260.611319:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573260.611360:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573260.611363:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573260.611368:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573260.611386:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088448:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573260.611390:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573260.611392:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611396:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.611397:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611399:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573260.611401:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573260.611402:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.611404:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088576:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573260.611406:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573260.611407:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573260.611409:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611410:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611412:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573260.611413:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.611416:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573260.611418:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573260.611420:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573260.611422:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810088576, portal 4 00000100:00000001:0.0:1788573260.611424:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573260.611425:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000040:0.0:1788573260.611429:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611433:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573260.611435:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573260.611436:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573260.611438:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810088576, offset 0 00000400:00000200:0.0:1788573260.611440:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611446:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611449:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573260.611451:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573260.611453:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573260.611454:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573260.611456:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611459:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540805:540805:256:4294967295] 192.168.202.148@tcp LPNI seq info [540770:540770:8:4294967295] 00000400:00000200:0.0:1788573260.611463:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611469:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573260.611474:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611477:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573260.611481:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573260.611485:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573260.611497:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611499:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573260.611502:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573260.611506:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573260.611509:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573260.611511:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573260.611513:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573260.611516:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611524:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611528:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611529:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573260.611531:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573260.611534:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573260.611536:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573260.611561:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308399777 00000100:00000001:0.0:1788573260.611564:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611566:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308399777 00000100:00000001:0.0:1788573260.611568:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611571:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573260.611573:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611575:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573260.611580:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573260/real 1788573260] req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573260.611589:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573260.611590:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573260.611592:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573260.611595:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573260.611597:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573260.611599:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573260.611602:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573260.611609:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611616:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573260.611617:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611619:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611620:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573260.611624:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611631:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573260.611651:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611659:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573260.611668:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611669:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611671:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611672:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573260.611674:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573260.611676:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573260.611679:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573260.611680:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573260.611683:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573260.611689:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088576:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573260.611691:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573260.611693:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611698:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.611699:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611700:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573260.611702:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573260.611703:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.611705:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088704:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573260.611707:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573260.611708:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573260.611710:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573260.611711:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611713:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573260.611714:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573260.611717:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573260.611719:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573260.611721:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573260.611723:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810088704, portal 4 00000100:00000001:0.0:1788573260.611729:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573260.611730:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923968 : -109714417627648 : ffff9c371f1c4e00) 00000100:00000040:0.0:1788573260.611734:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611739:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573260.611740:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573260.611742:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573260.611743:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810088704, offset 0 00000400:00000200:0.0:1788573260.611746:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611748:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611751:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573260.611753:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573260.611754:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573260.611756:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573260.611758:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573260.611760:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [540806:540806:256:4294967295] 192.168.202.148@tcp LPNI seq info [540771:540771:8:4294967295] 00000400:00000200:0.0:1788573260.611763:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573260.611767:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573260.611770:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611772:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573260.611774:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573260.611776:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 36 secs later) 00000800:00000100:0.0:1788573260.611784:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573260.611786:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573260.611787:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573260.611790:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573260.611792:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573260.611793:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573260.611795:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573260.611797:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611809:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611812:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573285 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611816:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611817:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573260.611820:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573260.611822:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573260.611824:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573260.611825:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308399787 00000100:00000001:0.0:1788573260.611827:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611828:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308399787 00000100:00000001:0.0:1788573260.611830:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308403422 00000100:00000001:0.0:1788573260.611831:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573260.611832:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.611833:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573260.611836:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573260/real 1788573260] req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573260.611840:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573260.611841:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573260.611842:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573260.611844:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573260.611845:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573260.611846:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573260.611848:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573260.611853:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611857:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573260.611861:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611862:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611863:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573260.611866:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611870:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573260.611872:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611876:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573260.611878:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611879:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573260.611880:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611881:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573260.611883:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573260.611884:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573260.611886:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573260.611887:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573260.611889:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573260.611894:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810088704:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573260.611896:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611897:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611899:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440810088320/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611903:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573260.611904:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611906:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573260.611907:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573260.611908:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573260.611911:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573260.611913:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573260.611914:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788573260.611916:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573260.611918:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573260.611919:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788573260.611920:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573260.611921:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611923:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611925:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440810088448/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611929:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573260.611930:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611931:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573260.611933:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573260.611933:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573260.611934:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573260.611935:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573260.611936:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233ec00. 02000000:00000001:0.0:1788573260.611937:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573260.611938:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573260.611939:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788573260.611940:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573260.611941:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611942:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611945:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440810088576/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611951:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573260.611952:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611954:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573260.611956:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573260.611956:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573260.611958:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573260.611959:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573260.611960:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233cc00. 02000000:00000001:0.0:1788573260.611963:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573260.611968:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573260.611970:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788573260.611972:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573260.611973:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.611975:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573260.611980:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4e00 x1875440810088704/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573285 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573260.611987:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573260.611988:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573260.611990:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573260.611993:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573260.611994:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573260.611996:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573260.611998:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573260.611999:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e800. 02000000:00000001:0.0:1788573260.612001:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573260.612002:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573260.612002:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4e00. 00000100:00000001:0.0:1788573260.612004:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573260.612005:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.612006:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.612009:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573260.612010:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.612012:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573260.612013:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573263.041835:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573263.041840:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573263.041844:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573264.065786:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573264.065789:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573264.065792:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573264.449748:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308403422, at 4308403456 00000100:00000010:0.0:1788573264.449765:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000020:00000001:0.0:1788573268.161869:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573268.161877:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573268.161883:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573270.785746:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308409759, at 4308409792 00000100:00000010:0.0:1788573270.785763:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788573273.281884:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573273.281888:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573273.281893:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573276.225746:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308415199, at 4308415232 00000100:00000010:0.0:1788573276.225762:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000100:00080000:0.0:1788573277.505803:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573277.505823:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573277.505825:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573277.505829:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573277.505833:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573277.505840:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13744 00000400:00000200:0.0:1788573277.505849:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573277.505855:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573277.505858:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573277.505860:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.505863:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.505865:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.505867:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573277.505870:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573277.505872:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573277.505874:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573277.505876:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573277.505878:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573277.505880:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573277.505882:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.505883:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.505884:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.505885:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573277.505911:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573277.505913:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573277.505919:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573277.505922:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573277.505927:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573277.505930:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.505932:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573277.505933:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573277.505940:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573277.505943:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573277.505947:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573277.505948:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.505951:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573277.505954:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573277.505957:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573277.505965:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573277.505971:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.505978:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810090880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573277.505987:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573277.505990:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.505994:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810090880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573277.506011:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573277.506014:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573277.506019:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573277.506020:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573277.506022:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573277.506024:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573277.506025:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13744 00000400:00000200:0.0:1788573277.506028:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573277.506039:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573277.506041:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573277.506043:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506045:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506047:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506048:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573277.506050:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573277.506051:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573277.506052:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573277.506054:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573277.506056:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573277.506058:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573277.506060:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506061:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506062:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506066:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573277.506068:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573277.506072:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506074:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573277.506076:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573277.506081:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573277.506083:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506087:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573277.506091:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573277.506096:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506098:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573277.506121:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573277.506124:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573277.506129:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573277.506131:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506134:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573277.506141:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573277.506145:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573277.506152:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573277.506159:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.506174:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810091008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573277.506188:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573277.506191:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.506197:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810091008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573277.506212:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573277.506217:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573277.506226:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573277.506229:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573277.506232:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573277.506237:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573277.506243:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13744 00000400:00000200:0.0:1788573277.506249:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573277.506253:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573277.506256:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573277.506262:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506267:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506268:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506270:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573277.506272:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573277.506274:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573277.506277:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573277.506279:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573277.506281:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573277.506283:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573277.506292:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506296:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506299:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506301:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573277.506302:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573277.506305:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506308:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573277.506309:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573277.506312:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573277.506317:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506320:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573277.506322:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573277.506325:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506328:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573277.506334:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573277.506339:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573277.506341:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573277.506342:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506344:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573277.506345:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573277.506347:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573277.506352:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573277.506356:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.506362:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810091136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573277.506371:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573277.506374:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.506377:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810091136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573277.506387:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573277.506390:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573277.506409:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573277.506412:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573277.506415:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573277.506418:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573277.506423:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13744 00000400:00000200:0.0:1788573277.506433:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573277.506438:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573277.506440:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573277.506443:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506444:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506449:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506454:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573277.506455:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573277.506457:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573277.506460:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573277.506462:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573277.506464:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573277.506466:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573277.506472:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506477:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506480:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506483:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573277.506485:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573277.506487:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506490:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573277.506491:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573277.506494:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573277.506498:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506502:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573277.506505:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573277.506513:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506515:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573277.506518:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573277.506520:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573277.506524:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573277.506525:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506529:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573277.506530:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573277.506532:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573277.506539:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573277.506542:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.506548:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810091264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573277.506556:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573277.506558:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.506560:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810091264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573277.506569:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573277.506572:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573277.506579:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573277.506581:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573277.506584:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573277.506589:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573277.506594:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13744 00000400:00000200:0.0:1788573277.506602:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573277.506610:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573277.506614:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573277.506619:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506622:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506624:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506633:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573277.506676:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573277.506678:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573277.506681:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573277.506683:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573277.506686:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573277.506689:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573277.506693:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506698:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573277.506701:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506704:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573277.506707:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573277.506709:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573277.506710:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573277.506711:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573277.506713:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573277.506716:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506721:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573277.506723:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573277.506725:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506728:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573277.506732:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573277.506734:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573277.506739:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573277.506741:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.506744:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573277.506747:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573277.506749:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573277.506754:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573277.506758:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.506766:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810091392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573277.506795:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573277.506797:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.506802:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810091392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573277.506817:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573277.506822:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13754) 00000020:00000001:0.0:1788573278.401875:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573278.401880:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573278.401884:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573281.345787:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308420318, at 4308420352 00000100:00000010:0.0:1788573281.345809:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000020:00000001:0.0:1788573283.521793:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573283.521797:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573283.521800:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573286.465744:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308425439, at 4308425472 00000100:00000010:0.0:1788573286.465758:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 10000000:00000001:0.0:1788573287.234034:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573287.234041:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9c80. 00000020:00000010:0.0:1788573287.234047:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573287.234082:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573287.234089:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573287.234096:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573287.234104:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573287.234110:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573287.234116:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788573287.234122:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3708e19800. 10000000:00000040:0.0:1788573287.234128:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573287.234130:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573287.234138:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573287.234145:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573287.234154:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573287.234160:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573287.234163:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573287.234168:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de7000. 20000000:00000010:0.0:1788573287.234172:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3708e19800. 00000020:00000010:0.0:1788573287.234175:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c54c00. 10000000:00000010:0.0:1788573287.234178:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9c80. 10000000:01000000:0.0:1788573287.234180:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573287.234183:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573287.234186:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573287.234188:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573287.234190:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573287.234193:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573287.234194:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573287.234197:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573287.234198:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573287.234200:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573287.234203:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573287.234204:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573287.234206:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573288.513734:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573288.513744:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573288.513750:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573288.513754:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573288.513756:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573288.513758:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573288.513761:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573288.513763:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573288.513766:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573288.513769:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573288.513772:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573288.513786:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573288.513793:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573288.513798:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573288.513802:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573288.513809:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573288.513820:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3708e19800. 00000100:00000001:0.0:1788573288.513845:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573288.513850:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573288.513853:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573288.513861:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46100. 00010000:00000001:0.0:1788573288.513865:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573288.513867:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788573288.513872:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573288.513875:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168aab to hash 00000020:00000001:0.0:1788573288.513876:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573288.513877:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573288.513880:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573288.513883:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573288.513886:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573288.513889:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573288.513897:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573288.513907:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573288.513912:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573288.513913:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573288.513919:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573288.513920:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573288.513922:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573288.513923:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000001:0.0:1788573288.513927:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573288.513931:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573288.513933:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573288.513935:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573288.513944:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573288.513959:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573288.513962:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573288.513986:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573288.513990:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573288.513994:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573288.513996:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573288.513997:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573288.513999:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573288.514001:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573288.514003:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573288.514005:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573288.514007:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573288.514009:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573288.514011:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573288.514017:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810092800:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573288.514021:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573288.514023:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573288.514025:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573288.514026:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573288.514029:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573288.514033:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573288.514035:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573288.514041:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573288.514042:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573288.514045:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573288.514051:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573288.514053:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573288.514054:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788573288.514056:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573288.514058:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788573288.514060:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514065:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573288.514068:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514071:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573288.514073:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514077:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573288.514078:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573288.514080:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514083:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573288.514084:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573288.514086:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573288.514089:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573288.514093:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573288.514095:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573288.514096:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514105:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573288.514106:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573288.514107:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573288.514108:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788573288.514110:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573288.514112:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573288.514115:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573288.514118:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573288.514124:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573288.514127:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573288.514133:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573288.514135:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573288.514137:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573288.514140:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573288.514142:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573288.514145:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573288.514148:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573288.514150:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573288.514152:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573288.514154:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168aab from hash 00010000:00000001:0.0:1788573288.514156:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573288.514158:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573288.514159:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573288.514160:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573288.514162:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573288.514164:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573288.514166:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573288.514167:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573288.514169:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573288.514176:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573288.514179:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573288.514183:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573288.514184:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573288.514185:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573288.514187:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573288.514188:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573288.514188:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573288.514189:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573288.514191:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168aab lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573288.514195:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46100 count: 0 00010000:00000001:0.0:1788573288.514203:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573288.514204:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573288.514206:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573288.514209:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6300 x1875440810092800/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573288.514214:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573288.514216:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573288.514217:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573288.514219:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573288.514221:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3708e19800. 02000000:00000001:0.0:1788573288.514224:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573288.514226:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573288.514227:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6300. 00000100:00000001:0.0:1788573288.514230:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573288.514231:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573288.514232:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573288.514234:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573288.514237:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573288.641853:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573288.641857:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573288.641860:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573292.353756:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308431327, at 4308431360 00000100:00000010:0.0:1788573292.353777:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788573293.761898:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573293.761915:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573293.761920:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573293.889778:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573293.889786:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573293.889788:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573293.889791:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573293.889794:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573293.889800:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13760 00000400:00000200:0.0:1788573293.889808:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573293.889813:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573293.889816:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573293.889819:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.889821:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.889823:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.889825:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573293.889828:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573293.889830:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573293.889832:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573293.889835:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573293.889836:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573293.889838:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573293.889840:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.889841:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.889842:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.889844:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573293.889846:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573293.889849:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573293.889866:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573293.889869:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573293.889871:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573293.889874:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.889875:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573293.889877:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573293.889882:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573293.889885:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573293.889888:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573293.889889:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.889892:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573293.889894:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573293.889898:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573293.889922:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573293.889930:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.889940:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810093568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573293.889959:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573293.889963:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.889968:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810093568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573293.889991:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573293.889993:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573293.889996:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573293.889997:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573293.889998:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573293.890000:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573293.890002:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13760 00000400:00000200:0.0:1788573293.890005:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573293.890007:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573293.890009:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573293.890011:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890020:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890022:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890024:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573293.890026:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573293.890027:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573293.890029:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573293.890031:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573293.890032:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573293.890041:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573293.890046:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890049:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890054:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890058:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573293.890063:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573293.890068:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890073:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573293.890077:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573293.890084:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573293.890092:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890095:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573293.890098:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573293.890103:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890105:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573293.890112:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573293.890115:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573293.890119:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573293.890121:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890124:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573293.890126:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573293.890130:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573293.890138:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573293.890144:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.890158:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810093696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573293.890168:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573293.890173:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.890178:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810093696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573293.890188:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573293.890193:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573293.890197:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573293.890198:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573293.890199:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573293.890201:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573293.890204:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13760 00000400:00000200:0.0:1788573293.890207:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573293.890212:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573293.890214:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573293.890221:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890223:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890228:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890230:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573293.890233:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573293.890236:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573293.890237:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573293.890239:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573293.890243:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573293.890246:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573293.890252:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890255:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890256:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890265:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573293.890268:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573293.890273:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890277:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573293.890279:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573293.890283:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573293.890288:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890292:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573293.890296:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573293.890300:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890303:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573293.890309:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573293.890314:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573293.890316:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573293.890318:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890322:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573293.890324:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573293.890328:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573293.890332:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573293.890337:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.890343:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810093824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573293.890352:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573293.890356:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.890361:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810093824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573293.890374:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573293.890379:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573293.890386:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573293.890388:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573293.890389:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573293.890408:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573293.890414:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13760 00000400:00000200:0.0:1788573293.890421:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573293.890425:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573293.890427:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573293.890430:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890435:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890437:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890442:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573293.890444:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573293.890446:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573293.890447:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573293.890451:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573293.890453:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573293.890455:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573293.890459:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890463:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890466:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890470:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573293.890474:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573293.890478:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890481:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573293.890483:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573293.890487:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573293.890491:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890495:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573293.890499:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573293.890502:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890505:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573293.890509:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573293.890522:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573293.890528:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573293.890530:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890535:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573293.890536:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573293.890538:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573293.890542:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573293.890546:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.890553:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810093952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573293.890565:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573293.890569:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.890572:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810093952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573293.890586:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573293.890592:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573293.890602:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573293.890605:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573293.890607:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573293.890611:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573293.890617:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13760 00000400:00000200:0.0:1788573293.890627:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573293.890634:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573293.890675:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573293.890681:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890685:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890689:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890693:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573293.890697:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573293.890699:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573293.890708:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573293.890712:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573293.890717:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573293.890721:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573293.890728:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890733:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573293.890738:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890740:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573293.890742:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573293.890743:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573293.890745:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573293.890748:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573293.890753:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573293.890756:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890760:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573293.890763:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573293.890768:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890772:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573293.890776:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573293.890780:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573293.890785:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573293.890788:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.890791:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573293.890794:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573293.890797:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573293.890802:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573293.890806:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.890811:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810094080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573293.890819:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573293.890824:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.890833:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810094080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573293.890844:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573293.890852:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13770) 00000100:00000040:0.0:1788573297.729726:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308436702, at 4308436736 00000100:00000010:0.0:1788573297.729743:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000800:00000200:0.0:1788573297.921909:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573298.881759:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573298.881763:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573298.881767:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573302.849750:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308441822, at 4308441856 00000100:00000010:0.0:1788573302.849766:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000020:00000001:0.0:1788573304.001873:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573304.001878:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573304.001882:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573304.129771:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573304.129798:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573304.129801:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573304.129805:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573304.129810:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573304.129817:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13770 00000400:00000200:0.0:1788573304.129828:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573304.129835:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573304.129838:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573304.129841:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.129844:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.129856:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.129859:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573304.129863:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573304.129865:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573304.129868:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573304.129871:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573304.129891:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573304.129894:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573304.129898:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.129900:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.129902:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.129904:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573304.129907:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573304.129911:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573304.129914:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573304.129917:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573304.129920:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573304.129923:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.129925:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573304.129928:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573304.129934:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573304.129937:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573304.129941:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573304.129943:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.129947:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573304.129949:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573304.129953:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573304.129960:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573304.129967:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.129977:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810094848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573304.129988:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573304.129992:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.129997:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810094848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573304.130023:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573304.130026:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573304.130036:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573304.130038:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573304.130039:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573304.130042:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573304.130045:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13770 00000400:00000200:0.0:1788573304.130050:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573304.130053:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573304.130057:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573304.130061:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130064:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130067:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130069:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573304.130072:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573304.130077:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573304.130080:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573304.130084:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573304.130089:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573304.130094:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573304.130102:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130108:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130112:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130117:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573304.130123:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573304.130130:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130133:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573304.130136:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573304.130142:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573304.130146:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130153:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573304.130159:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573304.130171:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130175:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573304.130181:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573304.130188:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573304.130192:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573304.130197:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130204:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573304.130205:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573304.130210:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573304.130219:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573304.130225:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.130239:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810094976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573304.130256:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573304.130261:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.130270:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810094976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573304.130283:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573304.130289:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573304.130297:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573304.130301:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573304.130306:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573304.130312:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573304.130316:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13770 00000400:00000200:0.0:1788573304.130326:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573304.130345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573304.130348:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573304.130356:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130370:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130377:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130390:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573304.130397:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573304.130401:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573304.130405:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573304.130408:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573304.130414:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573304.130418:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573304.130428:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130434:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130437:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130441:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573304.130447:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573304.130456:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130461:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573304.130467:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573304.130470:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573304.130475:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130477:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573304.130484:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573304.130490:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130495:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573304.130502:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573304.130507:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573304.130514:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573304.130518:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130528:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573304.130532:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573304.130536:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573304.130542:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573304.130547:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.130559:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810095104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573304.130616:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573304.130621:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.130632:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810095104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573304.130695:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573304.130700:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573304.130706:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573304.130710:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573304.130714:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573304.130722:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573304.130727:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13770 00000400:00000200:0.0:1788573304.130737:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573304.130746:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573304.130752:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573304.130760:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130765:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130770:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130776:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573304.130793:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573304.130794:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573304.130796:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573304.130800:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573304.130805:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573304.130810:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573304.130820:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130827:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.130833:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130839:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573304.130852:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573304.130864:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573304.130871:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573304.130876:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573304.130882:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573304.130886:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130891:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573304.130895:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573304.130903:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130909:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573304.130915:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573304.130919:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573304.130926:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573304.130929:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.130934:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573304.130938:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573304.130943:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573304.130954:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573304.130964:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.130974:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810095232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573304.130999:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573304.131005:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.131014:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810095232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573304.131032:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573304.131039:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573304.131047:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573304.131051:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573304.131055:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573304.131061:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573304.131075:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13770 00000400:00000200:0.0:1788573304.131086:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573304.131093:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573304.131097:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573304.131103:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.131106:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.131112:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.131117:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573304.131120:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573304.131124:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573304.131129:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573304.131133:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573304.131138:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573304.131145:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573304.131155:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.131159:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573304.131166:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573304.131172:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573304.131177:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573304.131185:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573304.131192:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573304.131196:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573304.131203:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573304.131208:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.131214:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573304.131219:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573304.131225:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.131230:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573304.131240:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573304.131246:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573304.131254:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573304.131267:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.131273:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573304.131278:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573304.131283:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573304.131292:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573304.131300:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.131312:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810095360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573304.131331:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573304.131334:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.131343:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810095360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573304.131369:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573304.131376:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13780) 00000100:00000040:0.0:1788573307.969786:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308446943, at 4308446976 00000100:00000010:0.0:1788573307.969803:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000020:00000001:0.0:1788573309.121824:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573309.121828:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573309.121832:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573309.249758:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573309.249768:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573309.249770:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573309.249774:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573309.249777:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573309.249784:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13775 00000400:00000200:0.0:1788573309.249792:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573309.249798:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573309.249800:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573309.249803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249805:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.249807:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249826:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573309.249829:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573309.249831:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573309.249833:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573309.249836:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573309.249838:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573309.249839:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573309.249841:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249843:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.249844:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249845:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573309.249848:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573309.249850:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249852:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573309.249854:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573309.249856:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573309.249858:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.249860:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573309.249862:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573309.249867:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573309.249869:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573309.249872:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573309.249873:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.249875:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573309.249877:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573309.249882:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573309.249888:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573309.249894:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.249901:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810095488/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573309.249908:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573309.249912:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.249919:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810095488/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573309.249940:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573309.249942:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573309.249944:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573309.249946:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573309.249947:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573309.249948:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573309.249950:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13775 00000400:00000200:0.0:1788573309.249953:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573309.249955:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573309.249956:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573309.249958:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249960:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.249962:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249964:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573309.249966:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573309.249967:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573309.249968:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573309.249970:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573309.249971:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573309.249972:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573309.249974:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249976:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.249978:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249982:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573309.249984:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573309.249989:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573309.249991:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573309.249998:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573309.250003:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573309.250008:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250011:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573309.250015:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573309.250020:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250025:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573309.250027:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573309.250031:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573309.250033:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573309.250036:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250038:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573309.250039:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573309.250043:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573309.250047:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573309.250052:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.250059:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810095616/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573309.250069:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573309.250073:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.250078:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810095616/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573309.250090:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573309.250093:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573309.250098:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573309.250100:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573309.250102:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573309.250106:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573309.250111:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13775 00000400:00000200:0.0:1788573309.250116:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573309.250119:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573309.250122:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573309.250136:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250138:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250140:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250146:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573309.250150:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573309.250153:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573309.250155:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573309.250156:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573309.250160:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573309.250164:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573309.250172:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250177:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250182:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250185:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573309.250189:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573309.250194:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250196:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573309.250198:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573309.250201:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573309.250205:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250208:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573309.250210:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573309.250212:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250215:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573309.250220:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573309.250223:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573309.250226:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573309.250228:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250231:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573309.250232:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573309.250234:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573309.250239:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573309.250247:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.250255:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810095744/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573309.250268:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573309.250271:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.250279:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810095744/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573309.250291:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573309.250293:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573309.250299:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573309.250302:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573309.250305:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573309.250309:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573309.250314:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13775 00000400:00000200:0.0:1788573309.250320:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573309.250335:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573309.250339:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573309.250346:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250350:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250354:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250360:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573309.250365:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573309.250368:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573309.250371:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573309.250376:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573309.250382:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573309.250388:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573309.250398:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250401:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250413:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250418:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573309.250426:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573309.250433:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250439:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573309.250444:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573309.250451:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573309.250458:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250463:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573309.250468:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573309.250473:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250477:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573309.250485:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573309.250491:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573309.250496:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573309.250499:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250502:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573309.250504:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573309.250507:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573309.250514:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573309.250518:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.250527:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810095872/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573309.250540:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573309.250544:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.250553:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810095872/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573309.250567:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573309.250574:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573309.250577:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573309.250579:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573309.250589:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573309.250594:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573309.250599:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13775 00000400:00000200:0.0:1788573309.250607:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573309.250612:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573309.250613:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573309.250618:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250620:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250622:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250623:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573309.250624:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573309.250625:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573309.250627:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573309.250629:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573309.250631:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573309.250634:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573309.250658:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250659:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573309.250660:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250661:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573309.250663:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573309.250665:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573309.250667:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573309.250668:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573309.250671:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573309.250676:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250680:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573309.250684:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573309.250690:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250693:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573309.250703:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573309.250707:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573309.250710:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573309.250712:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.250729:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573309.250731:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573309.250734:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573309.250740:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573309.250745:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.250753:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810096000/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573309.250771:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573309.250774:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.250781:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810096000/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573309.250794:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573309.250800:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13785) 00000100:00080000:0.0:1788573310.402285:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573310.402292:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573310.402295:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573310.402299:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573310.402303:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573310.402307:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13780 00000400:00000200:0.0:1788573310.402312:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573310.402317:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573310.402320:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573310.402322:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402325:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402327:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402328:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573310.402331:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573310.402338:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573310.402340:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573310.402342:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573310.402343:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573310.402345:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573310.402348:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402349:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402350:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402352:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573310.402354:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573310.402356:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402358:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573310.402359:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573310.402362:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573310.402364:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402365:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573310.402367:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573310.402369:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573310.402371:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573310.402373:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573310.402374:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402376:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573310.402377:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573310.402379:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573310.402383:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573310.402385:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.402390:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810096256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573310.402396:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573310.402398:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.402402:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810096256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573310.402427:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573310.402430:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573310.402432:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573310.402433:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573310.402435:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573310.402436:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573310.402438:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13780 00000400:00000200:0.0:1788573310.402441:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573310.402443:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573310.402445:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573310.402447:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402449:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402451:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402453:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573310.402454:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573310.402455:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573310.402457:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573310.402458:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573310.402460:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573310.402461:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573310.402466:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402470:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402474:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402476:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573310.402479:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573310.402486:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402490:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573310.402498:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573310.402504:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573310.402510:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402522:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573310.402529:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573310.402534:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402536:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573310.402540:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573310.402542:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573310.402547:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573310.402550:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402553:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573310.402554:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573310.402558:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573310.402564:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573310.402569:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.402575:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810096384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573310.402590:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573310.402598:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.402606:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810096384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573310.402621:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573310.402626:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573310.402632:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573310.402634:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573310.402676:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573310.402684:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573310.402689:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13780 00000400:00000200:0.0:1788573310.402720:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573310.402729:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573310.402735:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573310.402756:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402762:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402769:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402774:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573310.402780:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573310.402782:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573310.402784:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573310.402789:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573310.402792:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573310.402798:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573310.402803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402808:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.402813:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402818:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573310.402824:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573310.402832:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573310.402839:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573310.402844:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573310.402850:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573310.402856:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402859:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573310.402863:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573310.402868:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402873:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573310.402881:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573310.402887:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573310.402905:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573310.402908:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.402912:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573310.402916:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573310.402921:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573310.402928:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573310.402943:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.402955:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810096512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573310.402973:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573310.402978:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.402989:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810096512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573310.403015:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573310.403021:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573310.403029:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573310.403035:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573310.403040:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573310.403046:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573310.403052:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13780 00000400:00000200:0.0:1788573310.403073:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573310.403084:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573310.403089:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573310.403096:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403103:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.403109:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403115:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573310.403120:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573310.403121:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573310.403124:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573310.403130:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573310.403135:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573310.403144:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573310.403153:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403157:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.403172:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403179:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573310.403184:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573310.403191:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403198:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573310.403204:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573310.403211:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573310.403217:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403223:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573310.403229:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573310.403233:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403235:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573310.403240:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573310.403247:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573310.403253:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573310.403256:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403262:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573310.403266:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573310.403272:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573310.403281:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573310.403289:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.403300:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810096640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573310.403315:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573310.403320:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.403329:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810096640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573310.403349:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573310.403354:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573310.403362:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573310.403367:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573310.403377:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573310.403383:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573310.403389:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13780 00000400:00000200:0.0:1788573310.403398:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573310.403408:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573310.403414:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573310.403423:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403430:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.403438:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403444:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573310.403450:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573310.403453:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573310.403458:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573310.403464:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573310.403470:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573310.403477:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573310.403486:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403491:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573310.403498:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403504:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573310.403510:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573310.403519:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573310.403524:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573310.403529:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573310.403535:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573310.403542:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403548:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573310.403554:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573310.403561:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403565:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573310.403582:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573310.403588:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573310.403595:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573310.403600:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.403605:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573310.403609:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573310.403614:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573310.403623:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573310.403631:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.403664:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810096768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573310.403687:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573310.403691:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.403699:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810096768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573310.403746:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.403755:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13787) 00000100:00000040:0.0:1788573314.241763:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308453215, at 4308453248 00000100:00000010:0.0:1788573314.241783:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788573314.241837:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573314.241840:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573314.241843:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573319.361895:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573319.361899:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573319.361902:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573319.745777:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308458719, at 4308458752 00000100:00000010:0.0:1788573319.745795:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000100:00080000:0.0:1788573321.025756:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573321.025767:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573321.025770:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573321.025774:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573321.025778:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573321.025801:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13787 00000400:00000200:0.0:1788573321.025811:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573321.025818:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573321.025822:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573321.025826:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.025829:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.025831:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.025834:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573321.025838:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573321.025841:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573321.025844:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573321.025847:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573321.025850:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573321.025853:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573321.025856:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.025858:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.025860:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.025862:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573321.025865:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573321.025869:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573321.025872:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573321.025876:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573321.025879:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573321.025882:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.025885:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573321.025887:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573321.025894:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573321.025898:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573321.025902:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573321.025904:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.025907:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573321.025910:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573321.025918:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573321.025927:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573321.025935:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.025944:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810097536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573321.025956:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573321.025960:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.025965:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810097536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573321.025988:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573321.025991:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573321.025995:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573321.025997:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573321.025999:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573321.026002:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573321.026005:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13787 00000400:00000200:0.0:1788573321.026011:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573321.026015:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573321.026019:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573321.026025:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026029:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026035:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026040:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573321.026043:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573321.026047:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573321.026051:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573321.026055:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573321.026061:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573321.026075:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573321.026084:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026095:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026102:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026106:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573321.026111:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573321.026118:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026123:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573321.026126:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573321.026134:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573321.026139:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026144:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573321.026150:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573321.026155:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026158:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573321.026165:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573321.026170:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573321.026177:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573321.026182:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026187:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573321.026189:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573321.026193:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573321.026200:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573321.026207:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.026217:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810097664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573321.026229:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573321.026234:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.026241:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810097664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573321.026259:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573321.026262:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573321.026268:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573321.026277:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573321.026281:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573321.026286:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573321.026294:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13787 00000400:00000200:0.0:1788573321.026304:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573321.026313:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573321.026318:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573321.026326:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026333:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026338:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026343:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573321.026346:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573321.026348:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573321.026352:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573321.026357:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573321.026362:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573321.026367:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573321.026374:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026378:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026381:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026385:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573321.026388:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573321.026395:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026399:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573321.026405:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573321.026411:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573321.026416:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026420:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573321.026425:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573321.026430:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026443:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573321.026451:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573321.026457:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573321.026463:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573321.026466:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026470:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573321.026474:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573321.026478:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573321.026484:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573321.026492:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.026501:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810097792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573321.026529:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573321.026535:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.026545:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810097792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573321.026561:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573321.026566:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573321.026573:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573321.026576:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573321.026596:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573321.026599:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573321.026605:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13787 00000400:00000200:0.0:1788573321.026614:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573321.026621:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573321.026625:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573321.026633:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026673:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026678:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026684:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573321.026688:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573321.026698:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573321.026701:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573321.026705:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573321.026710:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573321.026715:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573321.026721:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026726:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026732:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026738:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573321.026744:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573321.026751:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026756:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573321.026760:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573321.026765:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573321.026770:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026775:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573321.026779:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573321.026785:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026792:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573321.026799:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573321.026804:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573321.026808:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573321.026811:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.026815:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573321.026818:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573321.026822:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573321.026827:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573321.026835:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.026845:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810097920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573321.026856:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573321.026860:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.026875:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810097920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573321.026892:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573321.026895:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573321.026899:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573321.026902:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573321.026906:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573321.026913:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573321.026918:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13787 00000400:00000200:0.0:1788573321.026925:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573321.026929:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573321.026931:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573321.026936:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026940:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026946:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026951:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573321.026957:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573321.026960:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573321.026964:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573321.026969:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573321.026975:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573321.026980:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573321.026986:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.026991:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573321.026996:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573321.027001:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573321.027006:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573321.027012:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573321.027018:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573321.027027:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573321.027033:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573321.027038:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.027043:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573321.027047:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573321.027053:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.027056:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573321.027063:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573321.027069:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573321.027074:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573321.027078:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573321.027082:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573321.027085:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573321.027088:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573321.027093:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573321.027099:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.027109:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810098048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573321.027125:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573321.027129:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.027139:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810098048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573321.027157:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.027165:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13797) 00000020:00000001:0.0:1788573324.481804:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573324.481808:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573324.481810:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573324.865747:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308463838, at 4308463872 00000100:00000010:0.0:1788573324.865762:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000100:00080000:0.0:1788573326.145763:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573326.145774:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573326.145777:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573326.145795:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573326.145799:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573326.145804:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13792 00000400:00000200:0.0:1788573326.145812:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573326.145817:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573326.145819:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573326.145821:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145824:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.145826:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145827:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573326.145830:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573326.145832:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573326.145834:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573326.145836:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573326.145838:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573326.145839:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573326.145841:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145842:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.145844:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145845:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573326.145848:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573326.145850:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145852:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573326.145854:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573326.145857:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573326.145858:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.145860:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573326.145861:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573326.145866:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573326.145868:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573326.145871:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573326.145875:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.145878:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573326.145880:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573326.145883:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573326.145890:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573326.145895:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.145902:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810098176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573326.145910:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573326.145913:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.145917:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810098176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573326.145935:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573326.145938:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573326.145940:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573326.145941:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573326.145942:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573326.145944:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573326.145946:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13792 00000400:00000200:0.0:1788573326.145949:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573326.145951:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573326.145952:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573326.145955:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145957:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.145959:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.145961:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573326.145962:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573326.145964:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573326.145965:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573326.145967:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573326.145968:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573326.145993:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573326.145999:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146002:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146006:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146010:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573326.146012:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573326.146017:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146019:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573326.146023:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573326.146038:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573326.146043:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146048:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573326.146053:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573326.146059:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146062:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573326.146066:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573326.146069:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573326.146071:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573326.146072:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146073:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573326.146074:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573326.146076:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573326.146081:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573326.146088:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.146095:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810098304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573326.146113:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573326.146116:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.146120:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810098304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573326.146134:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573326.146147:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573326.146151:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573326.146154:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573326.146156:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573326.146160:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573326.146164:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13792 00000400:00000200:0.0:1788573326.146172:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573326.146174:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573326.146176:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573326.146181:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146184:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146186:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146188:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573326.146191:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573326.146193:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573326.146196:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573326.146198:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573326.146201:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573326.146205:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573326.146210:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146214:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146217:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146221:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573326.146225:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573326.146231:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146236:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573326.146241:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573326.146245:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573326.146250:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146252:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573326.146259:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573326.146264:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146267:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573326.146271:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573326.146274:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573326.146276:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573326.146280:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146281:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573326.146283:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573326.146286:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573326.146292:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573326.146296:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.146304:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810098432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573326.146313:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573326.146315:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.146321:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810098432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573326.146335:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573326.146341:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573326.146344:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573326.146347:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573326.146350:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573326.146353:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573326.146358:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13792 00000400:00000200:0.0:1788573326.146364:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573326.146371:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573326.146374:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573326.146378:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146382:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146387:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146397:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573326.146400:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573326.146404:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573326.146406:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573326.146410:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573326.146413:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573326.146417:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573326.146421:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146424:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146425:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146427:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573326.146432:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573326.146434:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146437:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573326.146441:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573326.146445:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573326.146450:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146453:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573326.146456:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573326.146461:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146465:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573326.146472:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573326.146476:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573326.146478:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573326.146481:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146485:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573326.146489:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573326.146491:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573326.146496:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573326.146500:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.146506:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810098560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573326.146564:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573326.146569:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.146577:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810098560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573326.146605:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573326.146609:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573326.146614:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573326.146617:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573326.146620:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573326.146628:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573326.146633:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13792 00000400:00000200:0.0:1788573326.146676:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573326.146682:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573326.146687:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573326.146694:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146699:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146704:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146706:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573326.146708:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573326.146709:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573326.146711:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573326.146715:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573326.146718:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573326.146721:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573326.146729:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146734:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573326.146742:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146748:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573326.146761:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573326.146769:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573326.146775:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573326.146780:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573326.146786:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573326.146791:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146797:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573326.146802:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573326.146808:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146815:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573326.146823:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573326.146828:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573326.146833:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573326.146836:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573326.146839:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573326.146840:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573326.146843:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573326.146848:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573326.146855:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.146867:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810098688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573326.146897:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573326.146901:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.146908:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810098688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573326.146924:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.146930:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13802) 00000020:00000001:0.0:1788573329.601854:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573329.601860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573329.601864:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573329.985716:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308468958, at 4308468992 00000100:00000010:0.0:1788573329.985733:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788573334.721832:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573334.721834:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573334.721837:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573335.809954:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573336.257758:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308475231, at 4308475264 00000100:00000010:0.0:1788573336.257778:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 00000020:00000001:0.0:1788573339.841790:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573339.841794:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573339.841798:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573341.761792:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308480735, at 4308480768 00000100:00000010:0.0:1788573341.761812:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788573344.961860:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573344.961866:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573344.961870:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573346.881771:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308485854, at 4308485888 00000100:00000010:0.0:1788573346.881790:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000020:00000001:0.0:1788573350.081883:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573350.081889:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573350.081893:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573352.001747:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308490975, at 4308491008 00000100:00000010:0.0:1788573352.001762:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 10000000:00000001:0.0:1788573352.769738:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573352.769749:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9840. 00000020:00000010:0.0:1788573352.769757:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3708e19800. 00000020:00000010:0.0:1788573352.769796:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573352.769799:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573352.769803:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573352.769808:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573352.769812:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573352.769815:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573352.769818:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7600. 10000000:00000040:0.0:1788573352.769822:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573352.769824:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573352.769841:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573352.769845:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573352.769848:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573352.769851:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573352.769854:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573352.769857:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573352.769858:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7600. 00000020:00000010:0.0:1788573352.769860:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3708e19800. 10000000:00000010:0.0:1788573352.769862:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9840. 10000000:01000000:0.0:1788573352.769865:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573352.769868:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573352.769870:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573352.769871:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573352.769873:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573352.769875:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573352.769876:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573352.769879:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573352.769880:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573352.769882:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573352.769884:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573352.769885:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573352.769887:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573353.985705:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573353.985715:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573353.985720:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573353.985722:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573353.985724:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573353.985725:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573353.985728:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573353.985729:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573353.985732:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573353.985734:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573353.985736:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573353.985747:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573353.985752:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573353.985757:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573353.985778:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573353.985784:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573353.985792:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7600. 00000100:00000001:0.0:1788573353.985801:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573353.985807:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573353.985810:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573353.985821:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46400. 00010000:00000001:0.0:1788573353.985827:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573353.985831:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788573353.985838:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573353.985842:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168ac0 to hash 00000020:00000001:0.0:1788573353.985844:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573353.985846:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573353.985850:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573353.985863:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573353.985867:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573353.985871:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573353.985881:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573353.985895:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573353.985900:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573353.985902:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573353.985908:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573353.985910:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573353.985913:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573353.985914:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788573353.985920:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573353.985924:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573353.985926:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573353.985933:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573353.985944:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573353.985952:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573353.985956:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573353.985960:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573353.985963:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573353.985968:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573353.985970:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573353.985971:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573353.985974:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573353.985976:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573353.985978:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573353.985981:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573353.985983:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573353.985985:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573353.985988:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573353.985994:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810102784:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573353.985999:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573353.986001:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573353.986003:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573353.986005:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573353.986008:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573353.986012:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573353.986014:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573353.986020:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573353.986026:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573353.986029:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573353.986032:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573353.986034:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573353.986036:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788573353.986038:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573353.986041:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788573353.986044:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986049:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573353.986052:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986056:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573353.986058:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986062:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573353.986064:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573353.986066:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986069:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573353.986071:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573353.986074:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573353.986076:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573353.986081:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573353.986082:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573353.986084:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986092:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573353.986093:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573353.986094:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573353.986096:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788573353.986097:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573353.986098:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573353.986100:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573353.986102:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573353.986106:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573353.986108:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573353.986119:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573353.986121:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573353.986122:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573353.986124:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573353.986125:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573353.986128:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573353.986129:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573353.986131:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573353.986132:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573353.986134:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168ac0 from hash 00010000:00000001:0.0:1788573353.986136:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573353.986137:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573353.986138:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573353.986138:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573353.986142:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573353.986144:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573353.986146:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573353.986146:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573353.986148:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573353.986156:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573353.986160:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573353.986166:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573353.986167:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573353.986169:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573353.986171:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573353.986173:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573353.986174:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573353.986175:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573353.986178:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168ac0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573353.986187:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46400 count: 0 00010000:00000001:0.0:1788573353.986199:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573353.986201:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573353.986204:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573353.986208:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810102784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573353.986213:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573353.986216:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573353.986217:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573353.986220:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573353.986222:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7600. 02000000:00000001:0.0:1788573353.986226:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573353.986228:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573353.986230:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788573353.986233:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573353.986235:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573353.986236:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573353.986239:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573353.986243:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573355.201872:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573355.201878:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573355.201881:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573357.825767:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308496799, at 4308496832 00000100:00000010:0.0:1788573357.825798:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000020:00000001:0.0:1788573360.321863:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573360.321868:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573360.321871:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573363.265785:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308502238, at 4308502272 00000100:00000010:0.0:1788573363.265804:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788573365.441800:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573365.441806:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573365.441815:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573368.385750:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308507358, at 4308507392 00000100:00000010:0.0:1788573368.385764:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573369.537890:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573369.537894:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573369.537896:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573370.561847:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573370.561851:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573370.561854:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573373.505713:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308512479, at 4308512512 00000100:00000010:0.0:1788573373.505727:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 10000000:00000001:0.0:1788573374.273758:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573374.273770:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9840. 00000020:00000010:0.0:1788573374.273777:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7600. 00000020:00000010:0.0:1788573374.273782:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de6800. 00000040:00000010:0.0:1788573374.273785:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7000. 00000020:00000010:0.0:1788573374.273789:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573374.273793:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573374.273796:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573374.273806:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573374.273810:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c54c00. 10000000:00000040:0.0:1788573374.273814:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573374.273815:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573374.273832:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de6800. 00000040:00000010:0.0:1788573374.273839:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7000. 00000020:00000010:0.0:1788573374.273842:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573374.273845:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573374.273847:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573374.273850:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573374.273852:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573374.273854:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7600. 10000000:00000010:0.0:1788573374.273856:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9840. 10000000:01000000:0.0:1788573374.273859:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573374.273861:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573374.273863:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573374.273864:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573374.273866:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573374.273868:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573374.273870:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573374.273872:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573374.273873:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573374.273875:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573374.273877:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573374.273878:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573374.273880:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788573374.657880:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573374.657888:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573374.657893:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573374.721932:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573375.681898:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573375.681905:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573375.681908:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573376.129765:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573376.129771:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573376.129775:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573376.129777:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573376.129779:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573376.129780:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573376.129782:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573376.129799:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573376.129801:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573376.129803:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573376.129804:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573376.129812:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573376.129815:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573376.129819:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573376.129822:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573376.129828:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573376.129834:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c54c00. 00000100:00000001:0.0:1788573376.129840:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573376.129843:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573376.129845:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573376.129852:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46400. 00010000:00000001:0.0:1788573376.129855:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573376.129858:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788573376.129861:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573376.129864:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168ac7 to hash 00000020:00000001:0.0:1788573376.129865:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573376.129866:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573376.129869:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573376.129872:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573376.129875:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573376.129878:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573376.129886:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573376.129895:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573376.129900:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573376.129901:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573376.129909:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573376.129916:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573376.129918:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573376.129920:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788573376.129923:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573376.129926:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573376.129929:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573376.129930:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573376.129938:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573376.129945:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573376.129948:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573376.129953:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573376.129956:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573376.129960:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573376.129962:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573376.129963:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573376.129965:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573376.129966:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573376.129968:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573376.129970:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573376.129972:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573376.129974:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573376.129976:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573376.129981:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810106112:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573376.129986:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573376.129987:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573376.129992:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573376.129994:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573376.129997:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573376.130002:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573376.130003:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573376.130008:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573376.130009:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573376.130011:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573376.130013:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573376.130014:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573376.130016:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788573376.130017:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573376.130020:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788573376.130022:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130027:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573376.130029:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130033:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573376.130035:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130039:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573376.130040:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573376.130042:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130045:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573376.130046:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573376.130051:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573376.130053:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573376.130058:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573376.130060:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573376.130061:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130065:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573376.130066:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573376.130067:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573376.130068:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788573376.130069:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573376.130070:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573376.130072:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573376.130074:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573376.130077:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573376.130079:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573376.130083:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573376.130084:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573376.130085:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573376.130087:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573376.130089:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573376.130091:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573376.130092:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573376.130093:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573376.130094:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573376.130096:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168ac7 from hash 00010000:00000001:0.0:1788573376.130097:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573376.130102:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573376.130103:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573376.130104:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573376.130105:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573376.130107:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573376.130108:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573376.130109:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573376.130110:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573376.130111:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573376.130113:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573376.130116:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573376.130117:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573376.130118:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573376.130119:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573376.130120:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573376.130121:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573376.130122:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573376.130123:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ac7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573376.130127:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46400 count: 0 00010000:00000001:0.0:1788573376.130134:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573376.130135:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573376.130137:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573376.130139:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810106112/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573376.130144:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573376.130146:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573376.130147:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573376.130149:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573376.130151:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c54c00. 02000000:00000001:0.0:1788573376.130153:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573376.130155:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573376.130156:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788573376.130162:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573376.130163:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573376.130164:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573376.130166:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573376.130169:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573379.777879:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573379.777883:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573379.777887:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573379.969798:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308518943, at 4308518976 00000100:00000010:0.0:1788573379.969821:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000020:00000001:0.0:1788573380.801849:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573380.801855:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573380.801860:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573384.897782:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573384.897786:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573384.897789:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573385.281701:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308524255, at 4308524288 00000100:00000010:0.0:1788573385.281714:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000020:00000001:0.0:1788573385.921845:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573385.921848:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573385.921851:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573390.017867:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573390.017873:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573390.017877:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573390.401753:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308529374, at 4308529408 00000100:00000010:0.0:1788573390.401778:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573391.041804:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573391.041810:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573391.041815:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573395.137868:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573395.137874:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573395.137878:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573395.521783:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308534495, at 4308534528 00000100:00000010:0.0:1788573395.521805:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788573396.161870:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573396.161877:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573396.161882:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573396.289735:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573396.289747:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9840. 00000020:00000010:0.0:1788573396.289753:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573396.289757:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573396.289762:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573396.289766:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573396.289770:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573396.289773:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573396.289777:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de7200. 20000000:00000010:0.0:1788573396.289780:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7000. 10000000:00000040:0.0:1788573396.289784:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573396.289786:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573396.289790:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573396.289797:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573396.289799:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573396.289802:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573396.289805:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573396.289807:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de7200. 20000000:00000010:0.0:1788573396.289810:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7000. 00000020:00000010:0.0:1788573396.289812:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c54c00. 10000000:00000010:0.0:1788573396.289813:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9840. 10000000:01000000:0.0:1788573396.289816:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573396.289818:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573396.289820:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573396.289821:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573396.289822:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573396.289824:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573396.289826:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573396.289828:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573396.289829:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573396.289831:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573396.289846:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573396.289847:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573396.289849:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573397.441744:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573397.441752:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573397.441758:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573397.441761:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573397.441763:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573397.441765:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573397.441768:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573397.441770:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573397.441773:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573397.441776:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573397.441777:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573397.441786:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573397.441791:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573397.441794:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573397.441797:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573397.441801:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573397.441807:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7000. 00000100:00000001:0.0:1788573397.441813:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573397.441817:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573397.441819:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573397.441825:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46400. 00010000:00000001:0.0:1788573397.441828:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573397.441831:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788573397.441835:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573397.441837:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168ace to hash 00000020:00000001:0.0:1788573397.441838:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573397.441840:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573397.441842:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573397.441844:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573397.441847:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573397.441850:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573397.441874:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573397.441883:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573397.441887:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573397.441889:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573397.441893:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573397.441894:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573397.441896:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573397.441897:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788573397.441901:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573397.441904:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573397.441906:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573397.441908:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573397.441915:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573397.441922:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573397.441925:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573397.441929:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573397.441932:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573397.441936:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573397.441938:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573397.441939:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573397.441941:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573397.441942:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573397.441944:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573397.441967:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.441969:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573397.441970:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573397.441974:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573397.441980:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810109440:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573397.441983:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.441985:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573397.441987:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573397.441989:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573397.441991:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573397.441996:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573397.441997:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573397.442002:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573397.442003:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573397.442005:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573397.442008:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573397.442009:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573397.442011:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788573397.442012:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573397.442014:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788573397.442016:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442021:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573397.442024:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442028:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573397.442035:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442039:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573397.442041:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573397.442043:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442046:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573397.442047:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573397.442049:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573397.442051:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573397.442055:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573397.442057:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573397.442058:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442062:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573397.442063:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573397.442064:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573397.442065:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788573397.442066:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573397.442067:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573397.442069:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573397.442071:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573397.442075:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573397.442077:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573397.442081:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573397.442092:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573397.442094:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573397.442096:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573397.442098:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573397.442102:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573397.442104:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573397.442105:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573397.442107:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573397.442109:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168ace from hash 00010000:00000001:0.0:1788573397.442111:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573397.442112:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573397.442112:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573397.442113:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573397.442114:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573397.442115:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573397.442117:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573397.442117:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573397.442118:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573397.442119:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573397.442121:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573397.442125:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573397.442125:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573397.442126:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573397.442127:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573397.442128:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573397.442129:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573397.442129:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573397.442131:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ace lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573397.442135:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46400 count: 0 00010000:00000001:0.0:1788573397.442142:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573397.442143:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573397.442145:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573397.442151:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810109440/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573397.442156:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573397.442157:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573397.442159:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573397.442160:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573397.442162:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7000. 02000000:00000001:0.0:1788573397.442165:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573397.442167:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573397.442168:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788573397.442171:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573397.442172:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573397.442174:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573397.442175:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573397.442178:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573401.281779:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308540255, at 4308540288 00000100:00000010:0.0:1788573401.281810:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000100:00000040:0.0:1788573406.785764:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308545759, at 4308545792 00000100:00000010:0.0:1788573406.785780:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000100:00000040:0.0:1788573411.905755:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308550878, at 4308550912 00000100:00000010:0.0:1788573411.905774:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000800:00000200:0.0:1788573412.610019:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573412.610029:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573412.610039:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573412.610043:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573412.610045:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573417.025750:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308555999, at 4308556032 00000100:00000010:0.0:1788573417.025768:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 10000000:00000001:0.0:1788573417.793784:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573417.793807:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9840. 00000020:00000010:0.0:1788573417.793818:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7000. 00000020:00000010:0.0:1788573417.793826:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573417.793831:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573417.793858:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573417.793866:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573417.793872:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573417.793880:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573417.793886:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c54c00. 10000000:00000040:0.0:1788573417.793892:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573417.793895:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573417.793903:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573417.793914:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573417.793917:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573417.793922:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573417.793926:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573417.793943:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573417.793948:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573417.793952:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7000. 10000000:00000010:0.0:1788573417.793955:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9840. 10000000:01000000:0.0:1788573417.793959:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573417.793963:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573417.793966:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573417.793968:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573417.793971:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573417.793975:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573417.793977:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573417.793981:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573417.793983:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573417.793987:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573417.793990:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573417.793992:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573417.793995:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573419.521756:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573419.521764:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573419.521769:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573419.521771:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573419.521773:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573419.521775:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573419.521794:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573419.521795:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573419.521798:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573419.521808:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573419.521810:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573419.521818:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573419.521824:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573419.521829:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.521832:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.521837:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.521844:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c54c00. 00000100:00000001:0.0:1788573419.521851:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573419.521855:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573419.521858:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573419.521865:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46400. 00010000:00000001:0.0:1788573419.521869:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573419.521872:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788573419.521877:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573419.521880:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168ad5 to hash 00000020:00000001:0.0:1788573419.521882:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573419.521884:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573419.521886:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573419.521890:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573419.521893:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573419.521897:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573419.521908:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573419.521919:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573419.521925:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573419.521931:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573419.521937:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573419.521939:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612160 : -109714523939456 : ffff9c3718c61d80) 00000100:00000001:0.0:1788573419.521942:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573419.521943:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788573419.521948:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.521952:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.521954:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573419.521956:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573419.521997:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.522008:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573419.522012:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.522020:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.522023:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.522030:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573419.522032:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573419.522034:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.522036:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61d80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573419.522039:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573419.522041:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573419.522044:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573419.522046:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573419.522048:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573419.522052:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573419.522059:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810112768:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573419.522064:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573419.522071:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573419.522073:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573419.522075:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573419.522079:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.522086:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.522088:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61d80. 00000100:00000001:0.0:1788573419.522094:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573419.522096:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573419.522098:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573419.522102:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573419.522103:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.522106:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788573419.522108:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573419.522111:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788573419.522114:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522121:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573419.522125:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522131:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573419.522134:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522141:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573419.522142:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573419.522145:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522150:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573419.522158:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573419.522161:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573419.522164:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573419.522170:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573419.522172:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573419.522175:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522181:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573419.522182:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573419.522183:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.522185:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788573419.522186:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573419.522189:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573419.522192:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573419.522194:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573419.522200:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573419.522202:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573419.522208:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573419.522210:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573419.522211:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573419.522214:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573419.522216:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573419.522219:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573419.522220:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573419.522222:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573419.522223:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573419.522225:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168ad5 from hash 00010000:00000001:0.0:1788573419.522230:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573419.522231:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573419.522233:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573419.522234:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573419.522235:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573419.522237:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573419.522239:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573419.522240:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573419.522242:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573419.522243:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573419.522246:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573419.522251:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573419.522252:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573419.522254:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573419.522255:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573419.522256:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573419.522257:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573419.522258:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573419.522261:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168ad5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573419.522267:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46400 count: 0 00010000:00000001:0.0:1788573419.522276:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573419.522278:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573419.522280:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573419.522283:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810112768/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.522290:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573419.522292:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573419.522294:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573419.522296:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573419.522298:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c54c00. 02000000:00000001:0.0:1788573419.522301:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573419.522303:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573419.522308:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788573419.522311:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573419.522313:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573419.522314:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573419.522317:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573419.522320:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573423.361740:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308562335, at 4308562368 00000100:00000010:0.0:1788573423.361754:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00000040:0.0:1788573428.801699:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308567775, at 4308567808 00000100:00000010:0.0:1788573428.801714:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000400:00000100:0.0:1788573433.153916:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788573433.153955:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573433.153963:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573433.153966:0:2390:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788573433.153969:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573433.153972:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573433.153976:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573433.153979:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fd00. 00000400:00000200:0.0:1788573433.153983:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573433.153987:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573433.153992:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1aa0 00000400:00000010:0.0:1788573433.153995:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1aa0. 00000100:00000001:0.0:1788573433.153999:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573433.154008:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c5f80 x1875440810114176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154037:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573433.154041:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810114176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154049:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.154051:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573433.154059:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573433.154062:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f300. 00000400:00000200:0.0:1788573433.154079:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573433.154084:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573433.154089:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788573433.154091:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788573433.154097:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573433.154104:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c4380 x1875440810114304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154116:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573433.154121:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440810114304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154134:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.154139:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573433.154149:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573433.154154:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f800. 00000400:00000200:0.0:1788573433.154160:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573433.154169:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573433.154173:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1880 00000400:00000010:0.0:1788573433.154178:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1880. 00000100:00000001:0.0:1788573433.154183:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573433.154192:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c5880 x1875440810114432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154209:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573433.154234:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5880 x1875440810114432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154246:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.154251:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573433.154262:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573433.154268:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f400. 00000400:00000200:0.0:1788573433.154275:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573433.154282:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573433.154287:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf13b8 00000400:00000010:0.0:1788573433.154297:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf13b8. 00000100:00000001:0.0:1788573433.154305:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573433.154317:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c5180 x1875440810114560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154332:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573433.154339:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440810114560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154350:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.154352:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573433.154360:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573433.154366:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fa00. 00000400:00000200:0.0:1788573433.154373:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573433.154382:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573433.154387:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1000 00000400:00000010:0.0:1788573433.154391:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1000. 00000100:00000001:0.0:1788573433.154396:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573433.154403:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c5500 x1875440810114688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154412:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573433.154415:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810114688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573455 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573433.154422:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.154425:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788573433.921791:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308572894, at 4308572928 00000100:00000010:0.0:1788573433.921815:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000100:00000040:0.0:1788573439.041753:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308578016, at 4308578048 00000100:00000010:0.0:1788573439.041768:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000100:00000040:0.0:1788573445.377752:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308584351, at 4308584384 00000100:00000010:0.0:1788573445.377771:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000800:00000200:0.0:1788573450.497953:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573450.817726:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308589790, at 4308589824 00000100:00000010:0.0:1788573450.817748:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000100:00000040:0.0:1788573455.937782:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308594910, at 4308594944 00000100:00000010:0.0:1788573455.937801:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000100:00000040:0.0:1788573461.057773:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308600031, at 4308600064 00000100:00000010:0.0:1788573461.057794:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000100:00000040:0.0:1788573467.329732:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308606303, at 4308606336 00000100:00000010:0.0:1788573467.329752:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000100:00000040:0.0:1788573472.833777:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308611806, at 4308611840 00000100:00000010:0.0:1788573472.833795:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000100:00000040:0.0:1788573477.953758:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308616927, at 4308616960 00000100:00000010:0.0:1788573477.953774:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000100:00000001:0.0:1788573479.234058:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573479.234067:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573479.234072:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573479.234084:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.234101:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.234104:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.234109:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573479.234113:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573479.234117:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.234120:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810121472:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573479.234124:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573479.234128:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573479.234131:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.234133:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573479.234138:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573479.234141:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.234149:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573479.234154:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573479.234180:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573479.234189:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810121472, portal 25 00000100:00000001:0.0:1788573479.234195:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573479.234199:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573479.234208:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.234228:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573479.234234:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573479.234237:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573479.234243:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810121472, offset 0 00000400:00000200:0.0:1788573479.234252:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573479.234262:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.234271:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573479.234277:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573479.234285:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573479.234291:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573479.234296:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573479.234304:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541052:541052:256:4294967295] 192.168.202.148@tcp LPNI seq info [541017:541017:8:4294967295] 00000400:00000200:0.0:1788573479.234317:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.234339:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573479.234355:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.234454:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573479.234478:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573479.234482:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573479.234537:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.234547:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573479.234578:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573479.234594:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573479.234606:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573479.234610:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573479.234716:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573479.234742:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.234787:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.234801:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.234832:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.234837:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573479.234848:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.234860:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573479.234870:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308622047 00000100:00000010:0.0:1788573479.234878:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573479.234886:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573479.234897:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308622047 00000100:00000010:0.0:1788573479.234901:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116268a0. 00000100:00000001:0.0:1788573479.234905:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573479.234909:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573479.234916:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.234928:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573479.234938:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573479/real 1788573479] req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573479.234964:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573479.234972:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573479.234978:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573479.234986:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573479.234988:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573479.234996:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573479.235003:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573479.235028:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235053:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573479.235062:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235069:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235072:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573479.235080:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235114:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573479.235125:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235165:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573479.235180:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235183:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235185:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235188:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573479.235194:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573479.235198:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573479.235203:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573479.235214:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573479.235218:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573479.235228:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810121472:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573479.235232:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235234:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.235238:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810121472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235243:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573479.235246:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573479.235249:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573479.235252:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573479.235254:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573479.235261:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573479.235263:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573479.235265:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788573479.235269:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573479.235270:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573479.235272:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573479.235276:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573479.235277:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235279:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235286:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573479.235287:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235290:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573479.235292:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573479.235293:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573479.235296:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235302:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.235303:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235306:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573479.235308:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573479.235310:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.235312:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573479.235315:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573479.235318:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573479.235320:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235322:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573479.235325:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573479.235326:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.235330:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573479.235333:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573479.235336:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573479.235341:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810121600, portal 10 00000100:00000001:0.0:1788573479.235342:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573479.235344:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573479.235349:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235357:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573479.235359:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573479.235361:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573479.235363:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810121600, offset 0 00000400:00000200:0.0:1788573479.235367:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235372:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235377:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573479.235381:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573479.235384:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573479.235387:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573479.235389:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235392:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541053:541053:256:4294967295] 192.168.202.148@tcp LPNI seq info [541018:541018:8:4294967295] 00000400:00000200:0.0:1788573479.235396:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235402:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573479.235406:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235408:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573479.235411:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573479.235413:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573479.235422:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235423:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573479.235426:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573479.235428:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573479.235430:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573479.235431:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573479.235433:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573479.235436:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235442:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.235444:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235456:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235457:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573479.235460:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573479.235463:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573479.235465:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573479.235467:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308618265 00000100:00000001:0.0:1788573479.235469:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235470:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308618265 00000100:00000001:0.0:1788573479.235472:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235473:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573479.235474:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235476:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573479.235478:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573479/real 1788573479] req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573479.235483:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573479.235484:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573479.235485:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573479.235486:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573479.235487:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573479.235489:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573479.235491:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573479.235496:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235500:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573479.235501:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235502:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235503:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573479.235505:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235512:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573479.235515:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235519:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573479.235521:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235522:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235523:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235524:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573479.235525:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573479.235527:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573479.235529:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573479.235530:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573479.235532:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573479.235537:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573479.235540:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573479.235543:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235547:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.235548:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235549:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573479.235551:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573479.235552:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.235554:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121728:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573479.235555:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573479.235557:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573479.235561:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235562:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573479.235564:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573479.235566:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.235568:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573479.235570:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573479.235572:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573479.235574:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810121728, portal 10 00000100:00000001:0.0:1788573479.235576:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573479.235577:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573479.235581:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235585:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573479.235587:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573479.235588:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573479.235590:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810121728, offset 0 00000400:00000200:0.0:1788573479.235592:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235595:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235597:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573479.235599:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573479.235600:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573479.235602:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573479.235604:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235606:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541054:541054:256:4294967295] 192.168.202.148@tcp LPNI seq info [541019:541019:8:4294967295] 00000400:00000200:0.0:1788573479.235609:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235628:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573479.235631:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235634:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573479.235655:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573479.235657:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573479.235669:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235673:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573479.235675:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573479.235678:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573479.235680:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573479.235681:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573479.235683:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573479.235686:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235691:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.235693:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235698:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235699:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573479.235701:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573479.235704:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573479.235706:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573479.235707:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308618270 00000100:00000001:0.0:1788573479.235709:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235711:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308618270 00000100:00000001:0.0:1788573479.235713:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235714:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573479.235715:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235716:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573479.235718:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573479/real 1788573479] req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573479.235724:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573479.235725:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573479.235726:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573479.235727:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573479.235728:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573479.235735:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573479.235737:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573479.235743:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235747:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573479.235747:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235749:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235749:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573479.235752:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235756:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573479.235758:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235763:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573479.235764:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235765:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235766:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235767:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573479.235769:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573479.235770:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573479.235772:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573479.235773:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573479.235776:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573479.235780:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121728:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573479.235783:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573479.235788:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235793:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.235794:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235796:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573479.235797:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573479.235799:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.235800:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573479.235803:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573479.235804:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573479.235806:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235807:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573479.235809:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573479.235810:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.235812:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573479.235814:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573479.235817:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573479.235819:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810121856, portal 4 00000100:00000001:0.0:1788573479.235820:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573479.235821:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573479.235825:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235829:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573479.235831:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573479.235832:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573479.235834:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810121856, offset 0 00000400:00000200:0.0:1788573479.235836:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235838:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235841:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573479.235842:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573479.235844:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573479.235846:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573479.235847:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573479.235852:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541055:541055:256:4294967295] 192.168.202.148@tcp LPNI seq info [541020:541020:8:4294967295] 00000400:00000200:0.0:1788573479.235855:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.235860:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573479.235863:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235865:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573479.235867:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573479.235869:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573479.235876:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.235878:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573479.235880:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573479.235882:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573479.235884:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573479.235885:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573479.235886:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573479.235889:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235894:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.235896:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235900:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235901:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573479.235903:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573479.235905:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573479.235907:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573479.235908:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308618402 00000100:00000001:0.0:1788573479.235910:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235911:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308618402 00000100:00000001:0.0:1788573479.235913:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.235914:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573479.235918:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.235920:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573479.235922:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573479/real 1788573479] req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573479.235928:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573479.235928:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573479.235929:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573479.235931:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573479.235932:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573479.235933:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573479.235935:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573479.235940:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235945:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573479.235946:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235947:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235947:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573479.235949:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235953:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573479.235956:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235960:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573479.235961:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235962:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.235963:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.235964:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573479.235965:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573479.235967:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573479.235972:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573479.235973:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573479.235976:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573479.235981:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573479.235983:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573479.235986:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.235989:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.235990:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.235992:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573479.235993:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573479.235995:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.235996:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121984:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573479.235998:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573479.236000:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573479.236001:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573479.236002:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573479.236004:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573479.236006:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573479.236008:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573479.236010:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573479.236012:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573479.236014:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810121984, portal 4 00000100:00000001:0.0:1788573479.236015:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573479.236016:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788573479.236020:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236025:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573479.236029:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573479.236030:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573479.236032:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810121984, offset 0 00000400:00000200:0.0:1788573479.236035:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573479.236037:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.236040:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573479.236041:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573479.236043:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573479.236045:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573479.236046:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573479.236048:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541056:541056:256:4294967295] 192.168.202.148@tcp LPNI seq info [541021:541021:8:4294967295] 00000400:00000200:0.0:1788573479.236051:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573479.236055:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573479.236057:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.236059:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573479.236062:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573479.236063:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573479.236070:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573479.236071:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573479.236073:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573479.236075:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573479.236077:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573479.236078:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573479.236080:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573479.236082:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236087:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.236090:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236098:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.236099:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573479.236109:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573479.236112:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573479.236116:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573479.236118:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308618411 00000100:00000001:0.0:1788573479.236121:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.236123:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308618411 00000100:00000001:0.0:1788573479.236125:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308622047 00000100:00000001:0.0:1788573479.236127:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573479.236130:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.236132:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573479.236136:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573479/real 1788573479] req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573479.236145:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573479.236146:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573479.236148:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573479.236149:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573479.236150:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573479.236152:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573479.236154:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573479.236160:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236164:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573479.236165:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236166:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.236167:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573479.236169:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236173:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573479.236176:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236186:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573479.236188:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236189:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573479.236190:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236191:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573479.236192:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573479.236193:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573479.236196:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573479.236197:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573479.236200:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573479.236205:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810121984:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573479.236207:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236209:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.236211:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810121600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236215:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573479.236216:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573479.236217:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573479.236219:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573479.236220:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573479.236221:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573479.236222:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573479.236223:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573479.236225:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573479.236226:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573479.236227:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573479.236229:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573479.236230:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236234:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.236236:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810121728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236240:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573479.236241:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573479.236243:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573479.236244:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573479.236245:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573479.236245:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573479.236246:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573479.236247:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573479.236249:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573479.236249:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573479.236250:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573479.236251:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573479.236252:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236253:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.236255:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810121856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236258:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573479.236259:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573479.236260:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573479.236261:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573479.236262:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573479.236263:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573479.236264:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573479.236264:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573479.236266:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573479.236266:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573479.236267:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573479.236269:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573479.236270:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236271:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573479.236272:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810121984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573479.236282:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573479.236283:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573479.236285:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573479.236286:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573479.236287:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573479.236287:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573479.236288:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573479.236289:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573479.236291:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573479.236291:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573479.236292:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573479.236293:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573479.236294:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236295:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.236298:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573479.236299:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.236300:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573479.236301:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573483.073762:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308622047, at 4308622080 00000100:00000010:0.0:1788573483.073780:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573484.354031:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573484.354039:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573484.354045:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573484.354059:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354079:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.354084:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.354093:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573484.354100:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.354106:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.354114:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810122112:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573484.354124:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573484.354131:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573484.354144:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.354149:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573484.354158:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573484.354164:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.354178:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573484.354185:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573484.354198:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573484.354209:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810122112, portal 25 00000100:00000001:0.0:1788573484.354217:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573484.354230:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788573484.354244:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354263:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573484.354271:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573484.354279:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573484.354288:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810122112, offset 0 00000400:00000200:0.0:1788573484.354302:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573484.354319:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.354332:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573484.354342:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573484.354348:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573484.354352:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573484.354355:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573484.354359:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541057:541057:256:4294967295] 192.168.202.148@tcp LPNI seq info [541022:541022:8:4294967295] 00000400:00000200:0.0:1788573484.354364:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.354376:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573484.354391:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.354401:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573484.354415:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573484.354424:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573484.354460:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.354473:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573484.354494:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573484.354507:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573484.354517:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573484.354522:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573484.354529:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573484.354538:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354564:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.354572:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354592:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.354597:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573484.354626:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.354671:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573484.354680:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308627167 00000100:00000010:0.0:1788573484.354689:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573484.354700:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573484.354714:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308627167 00000100:00000010:0.0:1788573484.354723:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626600. 00000100:00000001:0.0:1788573484.354730:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573484.354735:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573484.354739:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.354745:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573484.354755:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573484/real 1788573484] req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573484.354774:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573484.354778:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573484.354784:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573484.354790:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573484.354795:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573484.354801:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573484.354816:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573484.354828:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354836:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573484.354839:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354842:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.354844:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573484.354849:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354858:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573484.354863:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354871:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573484.354875:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354877:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.354878:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354881:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573484.354884:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573484.354887:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573484.354892:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573484.354895:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573484.354899:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573484.354908:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810122112:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573484.354914:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354925:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.354929:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810122112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.354948:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573484.354951:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573484.354954:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573484.354958:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573484.354960:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573484.354962:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573484.354964:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573484.354966:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788573484.354971:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573484.354973:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573484.354974:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788573484.354978:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573484.354980:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354982:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.354990:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573484.354992:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.354994:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573484.354996:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573484.354998:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573484.355002:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355009:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.355011:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355013:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573484.355015:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.355017:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.355020:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122240:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573484.355023:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573484.355026:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573484.355027:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355030:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573484.355033:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573484.355035:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.355040:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573484.355052:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573484.355056:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573484.355059:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810122240, portal 10 00000100:00000001:0.0:1788573484.355062:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573484.355063:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788573484.355071:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355078:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573484.355080:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573484.355082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573484.355085:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810122240, offset 0 00000400:00000200:0.0:1788573484.355089:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355094:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355098:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573484.355101:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573484.355105:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573484.355107:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573484.355110:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355113:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541058:541058:256:4294967295] 192.168.202.148@tcp LPNI seq info [541023:541023:8:4294967295] 00000400:00000200:0.0:1788573484.355117:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355124:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573484.355129:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355132:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573484.355137:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573484.355140:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573484.355152:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355154:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573484.355157:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573484.355160:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573484.355164:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573484.355166:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573484.355174:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573484.355179:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355187:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.355190:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355198:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355200:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573484.355204:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.355209:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573484.355212:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573484.355213:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308623385 00000100:00000001:0.0:1788573484.355216:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355218:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308623385 00000100:00000001:0.0:1788573484.355219:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355221:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573484.355222:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355224:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573484.355226:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573484/real 1788573484] req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573484.355232:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573484.355233:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573484.355235:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573484.355237:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573484.355238:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573484.355240:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573484.355242:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573484.355248:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355260:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573484.355261:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355263:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355264:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573484.355266:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355271:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573484.355274:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355278:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573484.355280:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355281:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355282:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355283:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573484.355285:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573484.355287:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573484.355289:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573484.355291:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573484.355295:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573484.355304:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122240:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573484.355308:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573484.355312:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355319:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.355320:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355327:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573484.355330:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.355332:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.355335:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122368:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573484.355339:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573484.355341:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573484.355342:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355344:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573484.355347:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573484.355350:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.355354:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573484.355357:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573484.355360:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573484.355364:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810122368, portal 10 00000100:00000001:0.0:1788573484.355366:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573484.355367:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788573484.355373:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355378:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573484.355380:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573484.355382:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573484.355384:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810122368, offset 0 00000400:00000200:0.0:1788573484.355386:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355389:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355392:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573484.355395:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573484.355397:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573484.355399:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573484.355400:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355402:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541059:541059:256:4294967295] 192.168.202.148@tcp LPNI seq info [541024:541024:8:4294967295] 00000400:00000200:0.0:1788573484.355405:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355410:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573484.355418:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355420:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573484.355423:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573484.355425:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573484.355433:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355434:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573484.355437:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573484.355439:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573484.355441:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573484.355443:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573484.355445:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573484.355447:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355452:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.355454:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355459:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355460:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573484.355462:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.355464:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573484.355467:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573484.355468:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308623389 00000100:00000001:0.0:1788573484.355470:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355472:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308623389 00000100:00000001:0.0:1788573484.355473:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355474:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573484.355475:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355477:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573484.355479:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573484/real 1788573484] req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573484.355488:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573484.355489:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573484.355490:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573484.355492:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573484.355493:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573484.355494:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573484.355496:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573484.355502:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355507:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573484.355508:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355509:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355509:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573484.355512:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355516:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573484.355518:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355522:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573484.355523:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355524:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355525:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355526:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573484.355528:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573484.355529:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573484.355531:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573484.355532:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573484.355538:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573484.355543:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122368:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573484.355546:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573484.355556:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355561:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.355564:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355566:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573484.355567:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.355568:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.355570:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122496:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573484.355573:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573484.355574:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573484.355575:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355576:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573484.355579:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573484.355580:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.355583:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573484.355585:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573484.355588:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573484.355590:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810122496, portal 4 00000100:00000001:0.0:1788573484.355592:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573484.355593:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788573484.355598:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355613:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573484.355616:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573484.355619:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573484.355621:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810122496, offset 0 00000400:00000200:0.0:1788573484.355629:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355632:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355649:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573484.355652:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573484.355654:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573484.355655:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573484.355657:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355659:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541060:541060:256:4294967295] 192.168.202.148@tcp LPNI seq info [541025:541025:8:4294967295] 00000400:00000200:0.0:1788573484.355662:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355667:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573484.355670:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355673:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573484.355676:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573484.355677:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573484.355685:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355686:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573484.355688:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573484.355691:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573484.355693:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573484.355694:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573484.355696:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573484.355699:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355705:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.355707:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355711:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355712:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573484.355714:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.355716:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573484.355719:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573484.355724:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308623522 00000100:00000001:0.0:1788573484.355726:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355728:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308623522 00000100:00000001:0.0:1788573484.355730:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355731:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573484.355733:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355734:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573484.355737:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573484/real 1788573484] req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573484.355742:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573484.355743:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573484.355744:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573484.355746:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573484.355747:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573484.355749:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573484.355751:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573484.355756:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355760:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573484.355761:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355762:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355763:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573484.355765:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355769:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573484.355772:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355776:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573484.355781:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355782:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355783:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355784:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573484.355787:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573484.355788:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573484.355790:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573484.355792:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573484.355794:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573484.355799:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122496:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573484.355802:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573484.355804:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355808:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.355809:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355810:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573484.355812:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.355813:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.355814:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122624:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573484.355816:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573484.355817:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573484.355819:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.355820:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573484.355822:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573484.355823:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573484.355826:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573484.355827:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573484.355829:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573484.355831:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810122624, portal 4 00000100:00000001:0.0:1788573484.355836:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573484.355837:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788573484.355841:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355846:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573484.355847:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573484.355849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573484.355851:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810122624, offset 0 00000400:00000200:0.0:1788573484.355853:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355856:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355858:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573484.355860:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573484.355861:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573484.355863:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573484.355864:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573484.355867:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541061:541061:256:4294967295] 192.168.202.148@tcp LPNI seq info [541026:541026:8:4294967295] 00000400:00000200:0.0:1788573484.355869:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.355874:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573484.355876:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355878:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573484.355880:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573484.355882:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573484.355890:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573484.355891:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573484.355893:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573484.355896:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573484.355898:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573484.355899:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573484.355901:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573484.355903:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355916:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.355918:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355922:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355923:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573484.355926:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.355928:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573484.355930:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573484.355931:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308623531 00000100:00000001:0.0:1788573484.355933:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355935:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308623531 00000100:00000001:0.0:1788573484.355936:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308627167 00000100:00000001:0.0:1788573484.355938:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573484.355939:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.355940:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573484.355942:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573484/real 1788573484] req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573484.355947:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573484.355948:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573484.355949:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573484.355950:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573484.355951:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573484.355952:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573484.355954:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573484.355959:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355963:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573484.355967:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355968:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355969:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573484.355972:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355976:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573484.355979:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.355982:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573484.355984:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355985:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573484.355986:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.355988:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573484.355989:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573484.355991:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573484.355993:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573484.355994:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573484.355996:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573484.356001:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810122624:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573484.356003:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356004:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.356006:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440810122240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.356010:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573484.356011:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573484.356012:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788573484.356013:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573484.356014:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573484.356018:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573484.356020:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573484.356021:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233ec00. 02000000:00000001:0.0:1788573484.356023:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573484.356024:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573484.356025:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788573484.356027:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573484.356028:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356030:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.356032:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440810122368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.356036:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573484.356037:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573484.356038:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573484.356039:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573484.356040:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573484.356041:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573484.356042:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573484.356043:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788573484.356044:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573484.356045:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573484.356045:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788573484.356047:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573484.356047:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356048:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.356050:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440810122496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.356054:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573484.356055:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573484.356055:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573484.356057:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573484.356057:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573484.356058:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573484.356059:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573484.356060:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788573484.356061:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573484.356066:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573484.356067:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788573484.356069:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573484.356070:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356071:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573484.356074:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810122624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.356078:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573484.356078:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573484.356079:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573484.356080:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573484.356081:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573484.356082:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573484.356083:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573484.356084:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d400. 02000000:00000001:0.0:1788573484.356085:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573484.356086:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573484.356086:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788573484.356088:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573484.356089:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356090:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.356092:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573484.356093:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.356094:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573484.356095:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573484.994301:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573484.994307:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573484.994309:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573484.994313:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573484.994316:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573484.994321:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13956 00000400:00000200:0.0:1788573484.994328:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573484.994334:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573484.994337:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573484.994340:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573484.994350:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573484.994352:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573484.994354:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573484.994357:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573484.994359:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573484.994360:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573484.994363:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573484.994364:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573484.994366:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573484.994368:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573484.994369:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573484.994370:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573484.994371:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573484.994374:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573484.994376:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573484.994378:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573484.994380:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573484.994382:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573484.994384:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.994386:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573484.994388:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573484.994391:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573484.994393:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573484.994395:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573484.994397:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.994398:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573484.994399:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573484.994401:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573484.994406:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573484.994408:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.994414:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810122880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573484.994428:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573484.994430:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.994435:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810122880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573484.994455:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.994462:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13961) 00000100:00000040:0.0:1788573488.833729:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308627807, at 4308627840 00000100:00000010:0.0:1788573488.833744:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626cc0. 00000800:00000200:0.0:1788573489.410033:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573494.337728:0:267615:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308633311, at 4308633344 00000100:00000010:0.0:1788573494.337754:0:267615:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000400:00000100:0.0:1788573498.689973:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788573498.690047:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573498.690058:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573498.690062:0:2391:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788573498.690068:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573498.690073:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573498.690080:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573498.690085:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f500. 00000400:00000200:0.0:1788573498.690093:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573498.690099:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573498.690108:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1d48 00000400:00000010:0.0:1788573498.690112:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1d48. 00000100:00000001:0.0:1788573498.690119:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573498.690131:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70000 x1875440810123648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690162:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573498.690168:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810123648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690204:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573498.690206:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573498.690234:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573498.690237:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fe00. 00000400:00000200:0.0:1788573498.690241:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573498.690245:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573498.690248:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1660 00000400:00000010:0.0:1788573498.690251:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1660. 00000100:00000001:0.0:1788573498.690254:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573498.690259:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70380 x1875440810123776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690268:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573498.690271:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810123776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690279:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573498.690280:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573498.690290:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573498.690293:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fc00. 00000400:00000200:0.0:1788573498.690296:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573498.690301:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573498.690304:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1000 00000400:00000010:0.0:1788573498.690306:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1000. 00000100:00000001:0.0:1788573498.690309:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573498.690314:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec73480 x1875440810123904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690322:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573498.690325:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810123904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690332:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573498.690334:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573498.690344:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573498.690347:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f700. 00000400:00000200:0.0:1788573498.690350:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573498.690353:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573498.690356:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1880 00000400:00000010:0.0:1788573498.690358:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1880. 00000100:00000001:0.0:1788573498.690360:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573498.690365:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70e00 x1875440810124032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690374:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573498.690377:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810124032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690384:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573498.690386:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573498.690392:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788573498.690395:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f600. 00000400:00000200:0.0:1788573498.690398:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573498.690402:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573498.690405:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1aa0 00000400:00000010:0.0:1788573498.690407:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1aa0. 00000100:00000001:0.0:1788573498.690410:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573498.690415:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec72680 x1875440810124160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690424:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573498.690427:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810124160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573498.690435:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573498.690438:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788573499.457742:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308638431, at 4308638464 00000100:00000010:0.0:1788573499.457767:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00000040:0.0:1788573504.577763:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308643551, at 4308643584 00000100:00000010:0.0:1788573504.577804:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 10000000:00000001:0.0:1788573505.345678:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573505.345692:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9780. 00000020:00000010:0.0:1788573505.345700:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573505.345706:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573505.345712:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573505.345718:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573505.345724:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573505.345728:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573505.345733:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573505.345737:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7600. 10000000:00000040:0.0:1788573505.345742:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573505.345744:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573505.345750:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573505.345755:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573505.345758:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573505.345763:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573505.345767:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573505.345770:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573505.345772:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7600. 00000020:00000010:0.0:1788573505.345774:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c54c00. 10000000:00000010:0.0:1788573505.345777:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9780. 10000000:01000000:0.0:1788573505.345780:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573505.345782:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573505.345784:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573505.345785:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573505.345787:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573505.345790:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573505.345792:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573505.345794:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573505.345796:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573505.345798:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573505.345802:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573505.345813:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573505.345815:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788573505.857787:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573505.857802:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573505.857805:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573505.857809:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573505.857814:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573505.857821:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 13972 00000400:00000200:0.0:1788573505.857831:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573505.857838:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573505.857842:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573505.857845:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.857848:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.857850:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.857852:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573505.857856:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573505.857858:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573505.857860:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573505.857864:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573505.857866:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573505.857867:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573505.857870:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.857871:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.857873:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.857874:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573505.857877:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573505.857880:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573505.857881:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573505.857884:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573505.857887:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573505.857890:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.857920:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573505.857922:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573505.857929:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573505.857933:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573505.857937:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573505.857939:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.857942:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573505.857946:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573505.857951:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573505.857959:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573505.857966:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.857976:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810124928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573505.857989:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573505.857996:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858001:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810124928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573505.858021:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573505.858024:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573505.858027:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573505.858028:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573505.858029:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573505.858031:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573505.858033:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13972 00000400:00000200:0.0:1788573505.858036:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573505.858039:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573505.858041:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573505.858043:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858045:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858047:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858049:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573505.858055:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573505.858057:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573505.858058:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573505.858060:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573505.858064:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573505.858067:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573505.858072:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858074:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858076:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858077:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573505.858079:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573505.858080:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858084:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573505.858086:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573505.858093:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573505.858098:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858101:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573505.858107:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573505.858112:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858116:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573505.858120:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573505.858124:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573505.858127:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573505.858129:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858133:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573505.858136:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573505.858140:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573505.858144:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573505.858147:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.858153:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810125056/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573505.858168:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573505.858174:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858180:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810125056/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573505.858190:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573505.858192:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573505.858195:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573505.858201:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573505.858204:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573505.858207:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573505.858208:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13972 00000400:00000200:0.0:1788573505.858213:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573505.858218:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573505.858220:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573505.858222:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858227:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858228:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858230:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573505.858231:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573505.858232:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573505.858233:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573505.858236:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573505.858239:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573505.858240:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573505.858242:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858243:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858245:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858249:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573505.858252:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573505.858256:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858263:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573505.858266:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573505.858269:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573505.858274:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858276:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573505.858278:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573505.858280:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858282:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573505.858289:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573505.858295:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573505.858300:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573505.858303:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858308:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573505.858311:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573505.858314:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573505.858320:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573505.858327:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.858336:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810125184/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573505.858347:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573505.858350:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858356:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810125184/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573505.858368:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573505.858371:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573505.858376:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573505.858378:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573505.858379:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573505.858384:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573505.858387:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13972 00000400:00000200:0.0:1788573505.858395:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573505.858407:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573505.858409:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573505.858412:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858413:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858416:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858418:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573505.858422:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573505.858425:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573505.858428:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573505.858431:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573505.858435:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573505.858437:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573505.858443:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858447:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858449:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858451:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573505.858453:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573505.858456:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858459:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573505.858461:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573505.858463:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573505.858466:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858468:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573505.858472:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573505.858475:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858478:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573505.858485:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573505.858492:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573505.858506:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573505.858511:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858517:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573505.858526:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573505.858529:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573505.858537:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573505.858545:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.858553:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810125312/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573505.858591:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573505.858593:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858601:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810125312/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573505.858613:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573505.858616:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573505.858621:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573505.858622:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573505.858623:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573505.858626:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573505.858630:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 13972 00000400:00000200:0.0:1788573505.858667:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573505.858671:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573505.858674:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573505.858679:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858682:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858685:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858687:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573505.858689:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573505.858689:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573505.858692:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573505.858696:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573505.858698:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573505.858703:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573505.858715:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858720:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573505.858724:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858727:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573505.858729:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573505.858732:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573505.858737:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573505.858739:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573505.858744:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573505.858748:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858751:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573505.858753:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573505.858758:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858762:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573505.858768:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573505.858772:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573505.858778:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573505.858780:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.858784:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573505.858788:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573505.858792:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573505.858797:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573505.858804:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.858813:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810125440/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573505.858823:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573505.858825:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858834:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810125440/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573505.858852:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.858857:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13982) 10000000:00000001:0.0:1788573506.945803:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573506.945815:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573506.945822:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573506.945836:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573506.945839:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573506.945841:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573506.945845:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573506.945848:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573506.945851:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573506.945855:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573506.945858:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573506.945875:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573506.945881:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573506.945888:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.945892:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.945900:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.945911:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7600. 00000100:00000001:0.0:1788573506.945921:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573506.945928:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573506.945931:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573506.945953:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46700. 00010000:00000001:0.0:1788573506.945959:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573506.945964:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788573506.945971:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573506.945974:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168af1 to hash 00000020:00000001:0.0:1788573506.945977:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573506.945979:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788573506.945982:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788573506.945987:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573506.945991:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573506.945997:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573506.946021:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573506.946064:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573506.946072:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573506.946075:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573506.946094:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573506.946097:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609280 : -109714523942336 : ffff9c3718c61240) 00000100:00000001:0.0:1788573506.946100:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573506.946102:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788573506.946108:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.946113:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.946116:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573506.946118:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573506.946131:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.946141:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573506.946146:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.946155:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.946159:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.946167:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573506.946170:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573506.946172:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.946175:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61240 going to sleep for 0 seconds 00000100:00000001:0.0:1788573506.946178:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573506.946192:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573506.946196:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573506.946199:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573506.946201:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573506.946213:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573506.946223:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810125568:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573506.946230:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573506.946233:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573506.946236:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573506.946239:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573506.946243:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.946251:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.946254:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573506.946262:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573506.946264:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573506.946267:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573506.946271:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573506.946273:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.946276:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788573506.946279:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788573506.946282:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788573506.946296:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946305:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573506.946310:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946317:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573506.946320:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946338:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573506.946341:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573506.946344:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946361:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573506.946363:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573506.946366:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573506.946370:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573506.946378:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573506.946380:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573506.946383:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946391:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573506.946392:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573506.946394:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.946396:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788573506.946398:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788573506.946401:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788573506.946405:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573506.946408:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573506.946417:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573506.946419:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573506.946426:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573506.946434:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573506.946436:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573506.946440:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573506.946442:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573506.946446:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573506.946448:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573506.946450:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573506.946452:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573506.946456:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168af1 from hash 00010000:00000001:0.0:1788573506.946459:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573506.946460:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573506.946462:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573506.946463:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573506.946465:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573506.946467:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573506.946470:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573506.946472:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573506.946473:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573506.946475:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573506.946479:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573506.946486:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573506.946487:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573506.946489:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573506.946491:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573506.946493:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573506.946494:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573506.946495:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573506.946498:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168af1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573506.946507:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46700 count: 0 00010000:00000001:0.0:1788573506.946518:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573506.946520:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573506.946523:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573506.946533:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810125568/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.946542:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573506.946545:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573506.946547:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573506.946550:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573506.946554:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7600. 02000000:00000001:0.0:1788573506.946586:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573506.946589:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573506.946591:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788573506.946597:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573506.946598:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573506.946600:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573506.946603:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573506.946606:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573510.785751:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308649759, at 4308649792 00000100:00000010:0.0:1788573510.785779:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00000040:0.0:1788573515.841743:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308654814, at 4308654848 00000100:00000010:0.0:1788573515.841763:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000100:00000040:0.0:1788573520.961725:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308659935, at 4308659968 00000100:00000010:0.0:1788573520.961759:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000100:00000040:0.0:1788573526.081752:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308665054, at 4308665088 00000100:00000010:0.0:1788573526.081766:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000800:00000200:0.0:1788573527.297952:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 10000000:00000001:0.0:1788573527.361730:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573527.361744:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9740. 00000020:00000010:0.0:1788573527.361754:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7600. 00000020:00000010:0.0:1788573527.361762:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de6800. 00000040:00000010:0.0:1788573527.361768:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7000. 00000020:00000010:0.0:1788573527.361775:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573527.361780:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573527.361785:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573527.361793:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573527.361809:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c54c00. 10000000:00000040:0.0:1788573527.361817:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573527.361824:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573527.361834:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de6800. 00000040:00000010:0.0:1788573527.361844:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7000. 00000020:00000010:0.0:1788573527.361853:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573527.361860:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7200. 80000000:00000010:0.0:1788573527.361868:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573527.361876:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573527.361881:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573527.361888:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7600. 10000000:00000010:0.0:1788573527.361895:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9740. 10000000:01000000:0.0:1788573527.361902:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573527.361908:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573527.361912:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573527.361915:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573527.361917:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573527.361920:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573527.361923:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573527.361927:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573527.361932:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573527.361935:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573527.361939:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573527.361942:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573527.361947:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573528.897725:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573528.897732:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573528.897736:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573528.897739:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573528.897740:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573528.897741:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573528.897744:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573528.897745:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573528.897748:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573528.897750:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573528.897751:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573528.897775:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573528.897779:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573528.897783:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573528.897786:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573528.897791:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573528.897797:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c54c00. 00000100:00000001:0.0:1788573528.897803:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573528.897807:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573528.897809:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573528.897816:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46000. 00010000:00000001:0.0:1788573528.897819:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573528.897822:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30200. 00000020:00000001:0.0:1788573528.897825:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573528.897827:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30200 with handle 0x1441586093168af8 to hash 00000020:00000001:0.0:1788573528.897829:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573528.897830:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788573528.897833:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788573528.897835:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573528.897838:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573528.897841:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573528.897849:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573528.897858:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573528.897862:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573528.897863:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573528.897869:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573528.897870:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609280 : -109714523942336 : ffff9c3718c61240) 00000100:00000001:0.0:1788573528.897872:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573528.897873:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788573528.897880:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573528.897884:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573528.897886:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573528.897887:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573528.897896:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573528.897903:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573528.897906:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573528.897910:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573528.897913:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573528.897917:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573528.897919:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573528.897920:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573528.897922:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61240 going to sleep for 0 seconds 00000100:00000001:0.0:1788573528.897924:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573528.897926:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573528.897928:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573528.897930:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573528.897932:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573528.897934:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573528.897939:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810128896:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573528.897944:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573528.897945:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573528.897947:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573528.897949:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573528.897951:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573528.897962:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573528.897964:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573528.897969:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573528.897970:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573528.897972:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573528.897975:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573528.897976:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573528.897978:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=4 00000020:00000001:0.0:1788573528.897979:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788573528.897981:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00010000:0.0:1788573528.897984:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.897988:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573528.897991:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.897995:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573528.897997:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.898000:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573528.898002:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573528.898013:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.898017:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573528.898021:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573528.898023:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573528.898025:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573528.898037:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573528.898039:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573528.898040:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.898044:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573528.898045:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573528.898046:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573528.898047:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=5 00000020:00000001:0.0:1788573528.898048:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788573528.898049:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788573528.898051:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573528.898053:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573528.898058:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573528.898059:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573528.898063:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573528.898065:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573528.898066:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573528.898068:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573528.898069:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573528.898072:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573528.898073:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573528.898074:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573528.898075:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573528.898077:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30200 with handle 0x1441586093168af8 from hash 00010000:00000001:0.0:1788573528.898078:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573528.898079:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573528.898080:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573528.898083:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573528.898084:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573528.898086:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573528.898088:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573528.898089:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573528.898090:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573528.898091:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573528.898093:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573528.898096:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573528.898097:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573528.898098:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573528.898099:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573528.898100:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573528.898100:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573528.898101:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573528.898103:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168af8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573528.898108:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46000 count: 0 00010000:00000001:0.0:1788573528.898115:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573528.898116:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573528.898118:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573528.898121:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810128896/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573528.898125:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573528.898127:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573528.898128:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573528.898130:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573528.898132:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c54c00. 02000000:00000001:0.0:1788573528.898135:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573528.898137:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573528.898139:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788573528.898141:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573528.898142:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573528.898148:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573528.898150:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573528.898152:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573532.737768:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308671711, at 4308671744 00000100:00000010:0.0:1788573532.737793:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000100:00000040:0.0:1788573537.857720:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308676830, at 4308676864 00000100:00000010:0.0:1788573537.857736:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000100:00000040:0.0:1788573542.977744:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308681950, at 4308681984 00000100:00000010:0.0:1788573542.977760:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000100:00000040:0.0:1788573548.097723:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308687070, at 4308687104 00000100:00000010:0.0:1788573548.097732:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 10000000:00000001:0.0:1788573549.377740:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573549.377753:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9640. 00000020:00000010:0.0:1788573549.377760:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573549.377767:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573549.377774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573549.377780:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573549.377786:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573549.377792:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573549.377799:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de7200. 20000000:00000010:0.0:1788573549.377803:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7000. 10000000:00000040:0.0:1788573549.377807:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573549.377808:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573549.377812:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573549.377821:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573549.377826:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573549.377832:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7600. 80000000:00000010:0.0:1788573549.377837:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573549.377841:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de7200. 20000000:00000010:0.0:1788573549.377845:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7000. 00000020:00000010:0.0:1788573549.377849:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c54c00. 10000000:00000010:0.0:1788573549.377853:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9640. 10000000:01000000:0.0:1788573549.377866:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573549.377871:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573549.377876:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573549.377879:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573549.377882:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573549.377887:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573549.377889:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573549.377892:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573549.377894:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573549.377898:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573549.377900:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573549.377903:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573549.377906:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000040:0.0:1788573555.137779:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308694111, at 4308694144 00000100:00000010:0.0:1788573555.137804:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000100:00080000:0.0:1788573556.545807:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573556.545823:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573556.545826:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573556.545832:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573556.545839:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573556.545848:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14023 00000400:00000200:0.0:1788573556.545861:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573556.545870:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573556.545874:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573556.545878:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.545882:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.545885:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.545888:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573556.545894:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573556.545897:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573556.545900:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573556.545905:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573556.545907:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573556.545936:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573556.545941:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.545943:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.545945:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.545947:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573556.545951:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573556.545956:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573556.545959:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573556.545963:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573556.545967:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573556.545971:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.545974:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573556.545977:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573556.545986:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573556.545992:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573556.545997:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573556.545999:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546003:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573556.546007:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573556.546012:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573556.546023:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573556.546033:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.546045:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810132992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573556.546058:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573556.546062:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.546068:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810132992/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573556.546095:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573556.546099:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573556.546104:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573556.546111:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573556.546112:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573556.546116:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573556.546119:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14023 00000400:00000200:0.0:1788573556.546125:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573556.546129:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573556.546132:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573556.546136:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546140:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546145:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546148:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573556.546154:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573556.546158:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573556.546162:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573556.546166:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573556.546172:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573556.546178:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573556.546187:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546193:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546198:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546204:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573556.546211:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573556.546220:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546227:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573556.546231:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573556.546241:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573556.546245:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546249:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573556.546255:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573556.546263:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546274:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573556.546281:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573556.546286:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573556.546289:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573556.546292:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546295:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573556.546297:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573556.546300:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573556.546306:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573556.546313:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.546323:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810133120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573556.546344:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573556.546350:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.546357:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810133120/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573556.546373:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573556.546379:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573556.546383:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573556.546384:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573556.546387:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573556.546391:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573556.546395:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14023 00000400:00000200:0.0:1788573556.546405:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573556.546413:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573556.546421:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573556.546428:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546433:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546440:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546447:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573556.546453:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573556.546465:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573556.546469:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573556.546472:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573556.546476:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573556.546481:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573556.546500:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546506:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546510:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546516:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573556.546522:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573556.546529:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546537:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573556.546543:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573556.546549:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573556.546556:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546561:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573556.546565:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573556.546569:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546575:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573556.546595:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573556.546600:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573556.546604:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573556.546608:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546613:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573556.546616:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573556.546619:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573556.546627:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573556.546634:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.546710:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810133248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573556.546727:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573556.546732:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.546746:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810133248/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573556.546758:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573556.546763:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573556.546768:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573556.546772:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573556.546776:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573556.546782:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573556.546788:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14023 00000400:00000200:0.0:1788573556.546798:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573556.546805:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573556.546809:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573556.546816:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546822:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546827:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546833:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573556.546837:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573556.546839:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573556.546844:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573556.546850:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573556.546856:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573556.546859:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573556.546867:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546873:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.546880:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546885:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573556.546889:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573556.546892:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573556.546895:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573556.546906:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573556.546912:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573556.546916:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546920:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573556.546924:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573556.546928:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546932:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573556.546936:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573556.546940:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573556.546945:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573556.546947:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.546951:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573556.546954:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573556.546957:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573556.546964:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573556.546970:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.546981:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810133376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573556.547001:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573556.547004:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.547009:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810133376/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573556.547026:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573556.547033:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573556.547041:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573556.547044:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573556.547047:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573556.547050:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573556.547055:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14023 00000400:00000200:0.0:1788573556.547065:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573556.547073:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573556.547079:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573556.547093:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.547097:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.547104:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.547110:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573556.547115:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573556.547118:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573556.547121:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573556.547125:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573556.547128:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573556.547131:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573556.547138:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.547143:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573556.547146:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573556.547150:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573556.547152:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573556.547156:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573556.547162:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573556.547164:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573556.547168:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573556.547173:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.547174:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573556.547176:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573556.547181:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.547185:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573556.547189:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573556.547193:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573556.547198:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573556.547201:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.547203:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573556.547205:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573556.547208:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573556.547213:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573556.547222:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.547231:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810133504/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573556.547240:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573556.547242:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.547244:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810133504/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573556.547252:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.547257:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14033) 00000100:00000040:0.0:1788573560.385749:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308699359, at 4308699392 00000100:00000010:0.0:1788573560.385765:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000100:00080000:0.0:1788573561.665733:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573561.665742:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573561.665744:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573561.665747:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573561.665751:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573561.665756:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14028 00000400:00000200:0.0:1788573561.665764:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573561.665770:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573561.665773:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573561.665776:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665779:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.665781:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665784:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573561.665787:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573561.665789:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573561.665791:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573561.665795:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573561.665797:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573561.665799:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573561.665817:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665819:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.665820:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665822:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573561.665824:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573561.665826:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665828:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573561.665830:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573561.665833:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573561.665835:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.665836:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573561.665838:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573561.665842:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573561.665845:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573561.665848:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573561.665849:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.665852:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573561.665854:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573561.665857:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573561.665864:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573561.665869:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.665876:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810133632/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573561.665883:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573561.665886:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.665890:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810133632/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573561.665913:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573561.665915:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573561.665918:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573561.665919:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573561.665924:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573561.665926:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573561.665929:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14028 00000400:00000200:0.0:1788573561.665932:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573561.665934:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573561.665936:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573561.665938:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665939:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.665941:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665943:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573561.665945:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573561.665946:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573561.665947:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573561.665949:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573561.665950:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573561.665951:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573561.665954:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665955:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.665957:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665958:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573561.665959:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573561.665963:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573561.665967:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573561.665969:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573561.665974:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573561.665980:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.665982:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573561.665985:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573561.665989:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.665993:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573561.666000:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573561.666004:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573561.666008:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573561.666012:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666014:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573561.666016:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573561.666019:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573561.666023:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573561.666027:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.666036:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810133760/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573561.666045:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573561.666049:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.666055:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810133760/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573561.666065:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573561.666069:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573561.666073:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573561.666076:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573561.666077:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573561.666079:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573561.666081:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14028 00000400:00000200:0.0:1788573561.666088:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573561.666091:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573561.666092:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573561.666094:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666099:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666100:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666102:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573561.666104:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573561.666112:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573561.666117:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573561.666119:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573561.666121:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573561.666123:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573561.666130:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666132:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666134:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666137:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573561.666142:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573561.666147:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666150:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573561.666153:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573561.666156:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573561.666158:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666160:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573561.666163:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573561.666170:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666175:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573561.666183:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573561.666189:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573561.666196:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573561.666199:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666204:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573561.666206:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573561.666211:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573561.666217:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573561.666224:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.666234:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810133888/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573561.666250:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573561.666255:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.666269:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810133888/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573561.666290:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573561.666294:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573561.666301:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573561.666303:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573561.666306:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573561.666310:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573561.666312:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14028 00000400:00000200:0.0:1788573561.666339:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573561.666345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573561.666348:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573561.666350:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666352:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666356:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666359:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573561.666360:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573561.666362:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573561.666363:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573561.666364:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573561.666366:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573561.666368:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573561.666376:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666381:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666387:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666392:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573561.666395:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573561.666398:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666401:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573561.666415:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573561.666430:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573561.666434:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666435:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573561.666437:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573561.666440:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666444:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573561.666448:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573561.666451:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573561.666454:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573561.666455:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666457:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573561.666458:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573561.666461:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573561.666467:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573561.666472:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.666480:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810134016/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573561.666491:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573561.666494:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.666498:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810134016/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573561.666510:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573561.666514:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573561.666520:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573561.666522:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573561.666527:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573561.666532:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573561.666539:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14028 00000400:00000200:0.0:1788573561.666548:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573561.666556:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573561.666561:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573561.666572:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666576:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666580:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666599:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573561.666602:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573561.666605:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573561.666608:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573561.666612:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573561.666618:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573561.666624:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573561.666632:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666658:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573561.666661:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666664:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573561.666668:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573561.666675:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573561.666680:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573561.666684:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573561.666692:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573561.666697:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666699:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573561.666701:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573561.666707:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666712:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573561.666720:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573561.666727:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573561.666734:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573561.666737:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.666741:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573561.666743:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573561.666745:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573561.666754:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573561.666774:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.666786:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810134144/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573561.666807:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573561.666813:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.666824:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810134144/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573561.666852:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.666860:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14038) 00000100:00000040:0.0:1788573565.505748:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308704478, at 4308704512 00000100:00000010:0.0:1788573565.505765:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000800:00000200:0.0:1788573567.233989:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573567.233995:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573567.234006:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573567.234010:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573567.234012:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573570.625722:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308709599, at 4308709632 00000100:00000010:0.0:1788573570.625741:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 10000000:00000001:0.0:1788573571.393763:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573571.393778:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a90c0. 00000020:00000010:0.0:1788573571.393788:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3704de7000. 00000020:00000010:0.0:1788573571.393794:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573571.393798:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573571.393804:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573571.393810:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573571.393815:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573571.393821:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573571.393827:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3730c54c00. 10000000:00000040:0.0:1788573571.393832:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573571.393834:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573571.393841:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3704de7200. 00000040:00000010:0.0:1788573571.393859:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3704de7600. 00000020:00000010:0.0:1788573571.393862:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573571.393865:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de6800. 80000000:00000010:0.0:1788573571.393867:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573571.393870:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3708e19800. 20000000:00000010:0.0:1788573571.393872:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573571.393875:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3704de7000. 10000000:00000010:0.0:1788573571.393876:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a90c0. 10000000:01000000:0.0:1788573571.393879:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573571.393881:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573571.393884:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573571.393885:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573571.393886:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573571.393888:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573571.393890:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573571.393893:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573571.393895:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573571.393898:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573571.393901:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573571.393903:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573571.393905:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573573.185759:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573573.185769:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573573.185776:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573573.185779:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573573.185782:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573573.185785:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573573.185788:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573573.185791:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573573.185794:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573573.185798:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573573.185800:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573573.185812:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573573.185818:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573573.185824:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573573.185827:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573573.185846:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573573.185854:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3730c54c00. 00000100:00000001:0.0:1788573573.185861:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573573.185865:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573573.185868:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573573.185876:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46700. 00010000:00000001:0.0:1788573573.185881:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573573.185884:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788573573.185889:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573573.185892:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x1441586093168b06 to hash 00000020:00000001:0.0:1788573573.185894:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573573.185896:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573573.185899:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573573.185902:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573573.185906:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573573.185911:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573573.185922:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573573.185935:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573573.185941:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573573.185943:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573573.185949:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573573.185951:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609280 : -109714523942336 : ffff9c3718c61240) 00000100:00000001:0.0:1788573573.185954:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573573.185956:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788573573.185960:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573573.185965:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573573.185967:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573573.185969:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573573.185984:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573573.185994:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573573.185998:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573573.186006:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573573.186010:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573573.186018:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573573.186021:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573573.186023:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573573.186026:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61240 going to sleep for 0 seconds 00000100:00000001:0.0:1788573573.186028:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573573.186031:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573573.186035:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.186037:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573573.186040:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573573.186044:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573573.186055:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810135552:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573573.186060:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.186063:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573573.186065:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573573.186068:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573573.186072:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573573.186081:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573573.186083:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573573.186089:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573573.186091:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573573.186097:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573573.186101:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573573.186103:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573573.186105:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=4 00000020:00000001:0.0:1788573573.186108:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573573.186111:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00010000:0.0:1788573573.186115:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186123:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573573.186135:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186142:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573573.186146:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186152:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573573.186154:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573573.186157:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186164:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573573.186165:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573573.186168:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573573.186171:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573573.186178:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573573.186181:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573573.186184:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186203:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573573.186205:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573573.186206:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573573.186208:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=5 00000020:00000001:0.0:1788573573.186211:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573573.186213:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573573.186216:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573573.186219:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573573.186226:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573573.186228:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573573.186234:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573573.186237:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573573.186238:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573573.186241:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573573.186244:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573573.186247:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573573.186248:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573573.186250:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573573.186252:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573573.186255:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31400 with handle 0x1441586093168b06 from hash 00010000:00000001:0.0:1788573573.186257:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573573.186259:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573573.186260:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573573.186261:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573573.186263:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573573.186265:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573573.186268:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573573.186269:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573573.186271:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573573.186276:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573573.186279:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573573.186285:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573573.186286:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573573.186288:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573573.186289:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573573.186291:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573573.186292:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573573.186293:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573573.186296:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b06 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573573.186304:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46700 count: 0 00010000:00000001:0.0:1788573573.186313:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573573.186315:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573573.186317:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573573.186322:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810135552/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573573.186330:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573573.186333:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573573.186334:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573573.186337:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573573.186339:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3730c54c00. 02000000:00000001:0.0:1788573573.186343:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573573.186345:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573573.186347:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788573573.186351:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573573.186352:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573573.186354:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573573.186357:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573573.186360:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573577.025754:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308715999, at 4308716032 00000100:00000010:0.0:1788573577.025775:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000100:00000040:0.0:1788573582.401774:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308721374, at 4308721408 00000100:00000010:0.0:1788573582.401789:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 00000100:00000040:0.0:1788573587.521769:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308726494, at 4308726528 00000100:00000010:0.0:1788573587.521790:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000100:00000040:0.0:1788573592.641736:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308731615, at 4308731648 00000100:00000010:0.0:1788573592.641750:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 10000000:00000001:0.0:1788573593.409738:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573593.409755:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a90c0. 00000020:00000010:0.0:1788573593.409764:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3730c54c00. 00000020:00000010:0.0:1788573593.409770:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573593.409777:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573593.409783:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573593.409789:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573593.409794:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573593.409799:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573593.409802:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c3704de7600. 10000000:00000040:0.0:1788573593.409808:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573593.409810:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573593.409816:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3730c55000. 00000040:00000010:0.0:1788573593.409824:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3708e19800. 00000020:00000010:0.0:1788573593.409828:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ef000. 00000020:00000010:0.0:1788573593.409832:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c3704de7000. 80000000:00000010:0.0:1788573593.409836:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626cc0. 00040000:00000010:0.0:1788573593.409839:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c3704de6800. 20000000:00000010:0.0:1788573593.409841:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c3704de7600. 00000020:00000010:0.0:1788573593.409844:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3730c54c00. 10000000:00000010:0.0:1788573593.409847:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a90c0. 10000000:01000000:0.0:1788573593.409850:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573593.409853:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573593.409856:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573593.409858:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573593.409872:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573593.409875:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573593.409877:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573593.409880:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573593.409882:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573593.409884:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573593.409887:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573593.409888:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573593.409890:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573594.689740:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573594.689749:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573594.689755:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573594.689758:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573594.689761:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573594.689763:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573594.689766:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573594.689768:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573594.689771:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573594.689775:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573594.689776:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573594.689786:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573594.689790:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573594.689794:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573594.689796:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573594.689801:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573594.689808:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7600. 00000100:00000001:0.0:1788573594.689820:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573594.689824:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573594.689826:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573594.689833:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46700. 00010000:00000001:0.0:1788573594.689836:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573594.689839:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788573594.689843:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573594.689846:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x1441586093168b0d to hash 00000020:00000001:0.0:1788573594.689847:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573594.689849:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573594.689851:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573594.689867:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573594.689869:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573594.689873:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573594.689882:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573594.689894:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573594.689901:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573594.689904:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573594.689910:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573594.689913:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609280 : -109714523942336 : ffff9c3718c61240) 00000100:00000001:0.0:1788573594.689916:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573594.689918:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788573594.689923:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573594.689927:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573594.689930:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573594.689931:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573594.689943:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573594.689962:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573594.689967:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573594.689975:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573594.689980:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573594.689985:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573594.689987:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573594.689993:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573594.689996:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61240 going to sleep for 0 seconds 00000100:00000001:0.0:1788573594.689997:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573594.689999:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573594.690002:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573594.690004:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573594.690006:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573594.690009:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573594.690015:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810138880:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573594.690018:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573594.690020:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573594.690022:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573594.690023:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573594.690026:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573594.690031:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573594.690033:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573594.690038:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573594.690038:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573594.690040:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573594.690043:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573594.690044:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573594.690046:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=4 00000020:00000001:0.0:1788573594.690048:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573594.690050:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00010000:0.0:1788573594.690053:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690057:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573594.690060:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690068:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573594.690070:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690074:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573594.690075:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573594.690077:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690081:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573594.690082:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573594.690084:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573594.690086:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573594.690090:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573594.690091:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573594.690093:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690099:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573594.690100:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573594.690102:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573594.690103:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=5 00000020:00000001:0.0:1788573594.690105:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573594.690108:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788573594.690111:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573594.690114:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573594.690120:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573594.690130:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573594.690137:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573594.690139:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573594.690141:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573594.690145:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573594.690147:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573594.690150:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573594.690152:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573594.690154:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573594.690155:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573594.690159:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31400 with handle 0x1441586093168b0d from hash 00010000:00000001:0.0:1788573594.690161:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573594.690163:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573594.690164:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573594.690165:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573594.690167:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573594.690169:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573594.690172:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573594.690173:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573594.690175:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573594.690177:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573594.690180:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573594.690186:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573594.690187:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573594.690188:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573594.690190:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573594.690191:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573594.690193:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573594.690194:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573594.690197:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168b0d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573594.690204:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46700 count: 0 00010000:00000001:0.0:1788573594.690220:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573594.690222:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573594.690224:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573594.690229:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810138880/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573594.690237:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573594.690240:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573594.690241:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573594.690243:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573594.690246:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7600. 02000000:00000001:0.0:1788573594.690251:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573594.690253:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573594.690255:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788573594.690258:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573594.690260:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573594.690262:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573594.690264:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573594.690267:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573598.529722:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308737503, at 4308737536 00000100:00000010:0.0:1788573598.529738:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000100:00000040:0.0:1788573603.905743:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308742879, at 4308742912 00000100:00000010:0.0:1788573603.905761:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000020:00000001:0.0:1788573605.057877:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573605.057880:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573605.057883:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573605.121907:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573605.121912:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573605.121922:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573605.121926:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573605.121928:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573609.025730:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308747998, at 4308748032 00000100:00000010:0.0:1788573609.025743:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573610.177877:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573610.177882:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573610.177905:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573610.305784:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573610.305799:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573610.305802:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573610.305807:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573610.305813:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573610.305830:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14076 00000400:00000200:0.0:1788573610.305841:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573610.305850:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573610.305854:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573610.305858:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.305862:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.305864:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.305867:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573610.305872:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573610.305875:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573610.305878:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573610.305882:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573610.305885:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573610.305887:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573610.305891:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.305893:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.305895:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.305898:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573610.305902:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573610.305905:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573610.305908:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573610.305912:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573610.305916:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573610.305919:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.305946:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573610.305949:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573610.305957:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573610.305962:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573610.305967:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573610.305969:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.305972:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573610.305976:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573610.305981:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573610.305991:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573610.305999:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.306011:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810140928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573610.306023:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573610.306036:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.306041:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810140928/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573610.306066:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573610.306070:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573610.306073:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573610.306075:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573610.306077:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573610.306080:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573610.306083:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14076 00000400:00000200:0.0:1788573610.306088:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573610.306093:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573610.306096:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573610.306100:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306103:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306106:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306111:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573610.306122:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573610.306125:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573610.306130:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573610.306134:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573610.306138:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573610.306143:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573610.306151:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306157:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306163:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306168:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573610.306172:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573610.306178:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306184:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573610.306187:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573610.306196:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573610.306200:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306202:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573610.306208:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573610.306216:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306219:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573610.306225:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573610.306229:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573610.306232:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573610.306235:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306241:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573610.306245:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573610.306250:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573610.306258:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573610.306266:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.306290:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810141056/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573610.306305:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573610.306316:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.306325:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810141056/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573610.306342:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573610.306349:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573610.306358:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573610.306362:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573610.306365:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573610.306369:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573610.306373:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14076 00000400:00000200:0.0:1788573610.306392:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573610.306399:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573610.306405:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573610.306412:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306417:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306424:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306429:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573610.306435:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573610.306438:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573610.306440:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573610.306446:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573610.306450:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573610.306453:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573610.306461:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306466:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306472:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306477:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573610.306484:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573610.306498:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306510:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573610.306515:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573610.306519:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573610.306522:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306526:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573610.306531:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573610.306539:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306543:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573610.306554:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573610.306559:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573610.306564:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573610.306568:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306572:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573610.306575:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573610.306579:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573610.306588:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573610.306596:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.306609:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810141184/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573610.306681:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573610.306686:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.306696:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810141184/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573610.306711:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573610.306715:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573610.306726:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573610.306729:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573610.306733:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573610.306740:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573610.306745:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14076 00000400:00000200:0.0:1788573610.306752:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573610.306767:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573610.306772:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573610.306781:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306786:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306788:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306791:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573610.306794:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573610.306795:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573610.306798:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573610.306802:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573610.306806:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573610.306810:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573610.306820:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306826:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.306833:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306837:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573610.306842:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573610.306850:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573610.306859:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573610.306865:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573610.306870:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573610.306875:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306880:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573610.306886:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573610.306893:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306898:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573610.306905:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573610.306911:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573610.306916:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573610.306919:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.306922:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573610.306934:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573610.306939:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573610.306948:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573610.306957:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.306970:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810141312/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573610.306991:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573610.306995:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.307003:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810141312/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573610.307024:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573610.307029:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573610.307036:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573610.307041:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573610.307045:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573610.307053:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573610.307060:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14076 00000400:00000200:0.0:1788573610.307068:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573610.307078:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573610.307083:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573610.307088:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.307093:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.307097:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.307101:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573610.307105:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573610.307109:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573610.307112:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573610.307116:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573610.307121:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573610.307127:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573610.307139:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.307143:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573610.307147:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573610.307152:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573610.307155:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573610.307160:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573610.307164:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573610.307168:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573610.307172:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573610.307177:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.307183:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573610.307186:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573610.307191:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.307194:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573610.307199:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573610.307204:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573610.307209:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573610.307212:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.307215:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573610.307218:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573610.307222:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573610.307228:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573610.307237:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.307244:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810141440/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573610.307254:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573610.307257:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.307263:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810141440/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573610.307296:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573610.307301:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14087) 00000100:00000040:0.0:1788573614.145748:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308753118, at 4308753152 00000100:00000010:0.0:1788573614.145768:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788573615.297879:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573615.297883:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573615.297886:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573615.425749:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573615.425761:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573615.425763:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573615.425768:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573615.425773:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573615.425780:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14082 00000400:00000200:0.0:1788573615.425791:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573615.425797:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573615.425800:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573615.425803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.425813:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.425815:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.425817:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573615.425821:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573615.425823:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573615.425826:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573615.425829:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573615.425832:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573615.425834:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573615.425836:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.425837:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.425838:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.425839:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573615.425842:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573615.425844:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573615.425873:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573615.425876:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573615.425879:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573615.425882:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.425884:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573615.425886:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573615.425892:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573615.425896:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573615.425900:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573615.425902:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.425905:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573615.425907:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573615.425917:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573615.425926:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573615.425934:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.425942:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810141568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573615.425951:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573615.425956:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.425963:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810141568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573615.425992:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573615.425995:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573615.425998:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573615.425999:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573615.426000:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573615.426002:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573615.426004:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14082 00000400:00000200:0.0:1788573615.426007:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573615.426009:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573615.426010:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573615.426012:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426019:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426021:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426023:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573615.426025:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573615.426027:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573615.426028:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573615.426030:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573615.426032:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573615.426033:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573615.426036:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426037:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426042:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426044:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573615.426049:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573615.426053:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426055:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573615.426058:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573615.426064:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573615.426069:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426073:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573615.426079:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573615.426085:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426087:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573615.426092:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573615.426096:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573615.426100:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573615.426102:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426103:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573615.426104:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573615.426106:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573615.426110:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573615.426116:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.426128:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810141696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573615.426139:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573615.426142:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.426146:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810141696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573615.426167:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573615.426171:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573615.426174:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573615.426176:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573615.426179:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573615.426182:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573615.426184:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14082 00000400:00000200:0.0:1788573615.426187:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573615.426192:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573615.426196:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573615.426218:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426224:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426228:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426231:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573615.426234:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573615.426236:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573615.426238:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573615.426242:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573615.426247:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573615.426249:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573615.426254:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426257:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426261:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426268:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573615.426272:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573615.426278:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426281:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573615.426284:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573615.426294:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573615.426297:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426299:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573615.426303:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573615.426308:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426311:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573615.426315:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573615.426319:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573615.426323:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573615.426326:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426328:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573615.426330:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573615.426332:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573615.426338:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573615.426344:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.426352:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810141824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573615.426361:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573615.426366:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.426369:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810141824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573615.426379:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573615.426383:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573615.426389:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573615.426392:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573615.426394:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573615.426406:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573615.426409:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14082 00000400:00000200:0.0:1788573615.426415:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573615.426423:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573615.426426:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573615.426430:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426435:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426438:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426440:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573615.426444:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573615.426446:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573615.426449:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573615.426452:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573615.426455:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573615.426459:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573615.426465:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426469:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426474:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426479:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573615.426483:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573615.426484:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426486:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573615.426487:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573615.426489:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573615.426493:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426495:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573615.426498:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573615.426500:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426503:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573615.426507:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573615.426517:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573615.426524:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573615.426528:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426532:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573615.426534:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573615.426537:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573615.426545:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573615.426552:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.426563:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810141952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573615.426590:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573615.426594:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.426598:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810141952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573615.426607:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573615.426611:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573615.426618:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573615.426621:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573615.426623:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573615.426626:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573615.426629:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14082 00000400:00000200:0.0:1788573615.426652:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573615.426656:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573615.426658:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573615.426660:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426662:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426663:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426665:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573615.426667:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573615.426668:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573615.426676:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573615.426680:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573615.426683:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573615.426686:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573615.426693:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426698:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573615.426701:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426706:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573615.426710:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573615.426716:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573615.426718:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573615.426721:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573615.426724:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573615.426727:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426730:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573615.426732:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573615.426737:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426741:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573615.426747:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573615.426751:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573615.426755:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573615.426757:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.426759:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573615.426761:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573615.426764:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573615.426769:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573615.426774:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.426780:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810142080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573615.426791:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573615.426795:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.426800:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810142080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573615.426821:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573615.426828:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14092) 00000100:00000040:0.0:1788573620.353757:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308759327, at 4308759360 00000100:00000010:0.0:1788573620.353769:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000020:00000001:0.0:1788573620.417834:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573620.417838:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573620.417842:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573625.409713:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308764382, at 4308764416 00000100:00000010:0.0:1788573625.409727:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788573625.537861:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573625.537867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573625.537871:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573630.529744:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308769502, at 4308769536 00000100:00000010:0.0:1788573630.529763:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788573630.657856:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573630.657864:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573630.657868:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573631.809785:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573631.809797:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573631.809800:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573631.809804:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573631.809809:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573631.809818:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14098 00000400:00000200:0.0:1788573631.809827:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573631.809834:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573631.809837:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573631.809840:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.809843:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.809846:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.809848:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573631.809871:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573631.809874:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573631.809877:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573631.809880:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573631.809883:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573631.809885:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573631.809888:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.809890:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.809892:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.809894:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573631.809897:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573631.809901:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573631.809904:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573631.809907:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573631.809911:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573631.809914:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.809929:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573631.809935:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573631.809942:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573631.809946:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573631.809950:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573631.809951:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.809955:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573631.809958:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573631.809962:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573631.809985:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573631.809992:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.810001:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810144256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573631.810012:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573631.810017:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.810022:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810144256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573631.810063:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573631.810067:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573631.810070:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573631.810072:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573631.810074:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573631.810077:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573631.810080:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14098 00000400:00000200:0.0:1788573631.810085:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573631.810089:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573631.810092:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573631.810096:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810099:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810101:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810104:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573631.810106:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573631.810108:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573631.810111:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573631.810116:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573631.810122:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573631.810127:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573631.810134:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810140:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810146:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810151:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573631.810157:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573631.810163:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810168:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573631.810195:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573631.810204:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573631.810211:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810215:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573631.810222:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573631.810228:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810233:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573631.810240:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573631.810245:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573631.810249:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573631.810253:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810255:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573631.810258:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573631.810261:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573631.810270:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573631.810278:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.810290:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810144384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573631.810307:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573631.810311:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.810321:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810144384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573631.810338:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573631.810343:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573631.810350:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573631.810354:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573631.810357:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573631.810362:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573631.810370:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14098 00000400:00000200:0.0:1788573631.810379:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573631.810389:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573631.810394:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573631.810404:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810406:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810408:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810410:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573631.810413:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573631.810414:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573631.810416:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573631.810418:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573631.810420:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573631.810423:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573631.810426:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810428:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810429:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810432:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573631.810434:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573631.810436:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810439:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573631.810441:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573631.810444:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573631.810447:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810453:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573631.810458:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573631.810464:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810466:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573631.810470:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573631.810476:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573631.810484:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573631.810487:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810491:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573631.810493:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573631.810495:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573631.810511:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573631.810524:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.810536:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810144512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573631.810556:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573631.810561:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.810571:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810144512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573631.810583:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573631.810592:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573631.810600:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573631.810605:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573631.810608:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573631.810612:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573631.810615:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14098 00000400:00000200:0.0:1788573631.810620:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573631.810624:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573631.810626:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573631.810629:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810632:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810634:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810690:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573631.810694:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573631.810695:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573631.810697:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573631.810699:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573631.810702:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573631.810704:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573631.810713:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810719:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.810735:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810742:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573631.810749:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573631.810757:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573631.810761:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573631.810764:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573631.810770:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573631.810778:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810784:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573631.810789:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573631.810794:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810795:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573631.810801:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573631.810806:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573631.810812:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573631.810816:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.810819:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573631.810824:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573631.810829:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573631.810839:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573631.810845:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.810857:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810144640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573631.810885:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573631.810891:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.810900:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810144640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573631.810920:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573631.810926:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573631.810936:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573631.810939:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573631.810950:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573631.810957:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573631.810978:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14098 00000400:00000200:0.0:1788573631.810988:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573631.810998:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573631.811004:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573631.811012:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.811019:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.811025:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.811032:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573631.811038:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573631.811041:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573631.811045:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573631.811050:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573631.811055:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573631.811062:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573631.811071:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.811076:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573631.811083:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573631.811089:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573631.811093:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573631.811101:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573631.811107:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573631.811109:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573631.811113:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573631.811119:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.811124:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573631.811128:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573631.811133:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.811136:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573631.811151:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573631.811157:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573631.811164:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573631.811167:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.811172:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573631.811176:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573631.811181:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573631.811188:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573631.811196:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.811208:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810144768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573631.811226:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573631.811232:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.811239:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810144768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573631.811260:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573631.811267:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14108) 00000100:00000040:0.0:1788573635.649750:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308774623, at 4308774656 00000100:00000010:0.0:1788573635.649768:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000020:00000001:0.0:1788573635.777869:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573635.777873:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573635.777877:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573640.897851:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573640.897856:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573640.897860:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573641.793750:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308780767, at 4308780800 00000100:00000010:0.0:1788573641.793771:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000800:00000200:0.0:1788573643.009973:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573646.017899:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573646.017903:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573646.017907:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573646.913780:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308785887, at 4308785920 00000100:00000010:0.0:1788573646.913805:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788573651.137863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573651.137868:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573651.137871:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573652.033734:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308791006, at 4308791040 00000100:00000010:0.0:1788573652.033755:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788573656.257863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573656.257867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573656.257870:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573657.153740:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308796127, at 4308796160 00000100:00000010:0.0:1788573657.153754:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 10000000:00000001:0.0:1788573660.417814:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573660.417826:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573660.417833:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573660.417836:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573660.417838:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573660.417840:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573660.417844:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573660.417846:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573660.417849:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573660.417853:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573660.417855:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573660.417870:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573660.417877:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573660.417884:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573660.417889:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573660.417896:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573660.417906:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c3704de7600. 00000100:00000001:0.0:1788573660.417925:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573660.417932:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573660.417936:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573660.417947:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3713c46900. 00010000:00000001:0.0:1788573660.417952:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573660.417957:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30a00. 00000020:00000001:0.0:1788573660.417963:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573660.417967:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30a00 with handle 0x1441586093168b22 to hash 00000020:00000001:0.0:1788573660.417970:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573660.417993:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573660.417997:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573660.418002:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573660.418006:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573660.418012:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573660.418035:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573660.418051:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573660.418058:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573660.418061:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573660.418070:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573660.418073:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609280 : -109714523942336 : ffff9c3718c61240) 00000100:00000001:0.0:1788573660.418076:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.418078:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788573660.418084:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573660.418090:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.418093:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573660.418095:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.418110:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.418121:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573660.418127:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.418141:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573660.418147:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.418155:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573660.418171:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573660.418173:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.418177:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61240 going to sleep for 0 seconds 00000100:00000001:0.0:1788573660.418180:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573660.418183:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573660.418188:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.418191:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.418193:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.418199:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.418209:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810148864:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573660.418216:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.418219:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573660.418221:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573660.418224:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.418228:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.418237:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573660.418240:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61240. 00000100:00000001:0.0:1788573660.418248:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573660.418250:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573660.418254:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573660.418257:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573660.418260:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573660.418263:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=4 00000020:00000001:0.0:1788573660.418265:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573660.418268:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00010000:0.0:1788573660.418272:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418280:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573660.418300:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418308:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573660.418312:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418319:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573660.418322:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573660.418325:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418332:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573660.418334:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573660.418337:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573660.418341:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573660.418348:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573660.418351:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573660.418355:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418361:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573660.418363:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573660.418365:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573660.418367:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788573660.418369:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573660.418372:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788573660.418375:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573660.418378:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573660.418407:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573660.418411:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573660.418417:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573660.418420:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573660.418422:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573660.418426:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573660.418428:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573660.418432:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573660.418435:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573660.418437:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573660.418439:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573660.418475:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30a00 with handle 0x1441586093168b22 from hash 00010000:00000001:0.0:1788573660.418479:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573660.418480:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573660.418482:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573660.418483:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573660.418491:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573660.418494:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573660.418497:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573660.418498:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573660.418500:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573660.418502:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573660.418506:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573660.418513:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573660.418514:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573660.418516:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573660.418518:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573660.418520:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573660.418521:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573660.418522:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573660.418532:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168b22 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573660.418541:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3713c46900 count: 0 00010000:00000001:0.0:1788573660.418552:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573660.418555:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573660.418558:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.418564:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440810148864/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.418572:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573660.418577:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.418579:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573660.418582:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.418585:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c3704de7600. 02000000:00000001:0.0:1788573660.418592:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.418594:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.418597:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788573660.418602:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.418603:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573660.418606:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573660.418608:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573660.418612:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573661.377858:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573661.377862:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573661.377866:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573664.257773:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308803232, at 4308803264 00000100:00000010:0.0:1788573664.257800:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788573666.497847:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573666.497851:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573666.497853:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573669.441738:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308808414, at 4308808448 00000100:00000010:0.0:1788573669.441757:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788573671.617854:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573671.617860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573671.617888:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573674.561786:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308813534, at 4308813568 00000100:00000010:0.0:1788573674.561807:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000020:00000001:0.0:1788573676.737897:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573676.737913:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573676.737916:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573679.681777:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308818654, at 4308818688 00000100:00000010:0.0:1788573679.681796:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 10000000:00000001:0.0:1788573680.449741:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573680.449795:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9500. 00000020:00000010:0.0:1788573680.449805:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788573680.449810:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788573680.449813:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788573680.449817:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573680.449821:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788573680.449825:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573680.449829:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788573680.449832:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ba00. 10000000:00000040:0.0:1788573680.449837:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573680.449839:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573680.449843:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788573680.449849:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788573680.449851:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573680.449854:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788573680.449856:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573680.449858:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788573680.449860:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788573680.449861:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788573680.449863:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9500. 10000000:01000000:0.0:1788573680.449866:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573680.449868:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573680.449870:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573680.449886:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573680.449889:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573680.449892:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573680.449895:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573680.449898:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573680.449900:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573680.449903:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573680.449905:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573680.449907:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573680.449910:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:0.0:1788573680.961980:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.961987:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.961992:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.962005:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962026:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.962030:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.962035:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.962040:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.962045:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.962054:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810151552:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573680.962060:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.962066:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.962069:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.962071:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.962077:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573680.962081:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.962088:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573680.962095:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573680.962106:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573680.962116:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810151552, portal 25 00000100:00000001:0.0:1788573680.962120:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.962122:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573680.962130:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962161:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.962166:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573680.962171:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573680.962176:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810151552, offset 0 00000400:00000200:0.0:1788573680.962185:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573680.962195:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.962204:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573680.962213:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.962219:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573680.962225:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.962230:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573680.962237:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541278:541278:256:4294967295] 192.168.202.148@tcp LPNI seq info [541243:541243:8:4294967295] 00000400:00000200:0.0:1788573680.962241:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.962247:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.962256:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.962276:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573680.962286:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573680.962290:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573680.962321:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.962325:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573680.962333:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.962343:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.962351:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573680.962355:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573680.962360:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.962366:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962383:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.962389:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962416:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.962419:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.962427:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.962434:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573680.962437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308823774 00000100:00000010:0.0:1788573680.962446:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573680.962454:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573680.962465:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308823774 00000100:00000010:0.0:1788573680.962472:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626e40. 00000100:00000001:0.0:1788573680.962478:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573680.962485:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.962491:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.962495:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.962504:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.962523:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.962527:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.962530:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573680.962536:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573680.962541:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573680.962548:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.962555:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.962578:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962589:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.962593:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962598:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.962601:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.962606:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962630:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.962670:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962686:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.962695:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962702:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.962707:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962713:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.962718:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.962728:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.962737:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573680.962740:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.962752:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.962779:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810151552:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573680.962792:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962796:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.962803:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810151552/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962823:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.962829:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.962832:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573680.962839:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.962842:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.962845:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573680.962849:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.962853:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573680.962859:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.962880:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.962884:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573680.962890:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.962895:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962902:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.962912:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.962917:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.962923:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.962940:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.962944:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.962953:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.962981:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.962984:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.962989:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.962993:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.962997:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.963005:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151680:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.963014:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.963017:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.963020:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.963023:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.963030:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573680.963035:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.963040:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573680.963043:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573680.963048:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573680.963052:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810151680, portal 10 00000100:00000001:0.0:1788573680.963055:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.963056:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573680.963065:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963072:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.963074:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573680.963083:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573680.963087:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810151680, offset 0 00000400:00000200:0.0:1788573680.963092:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963096:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963100:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573680.963104:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.963106:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573680.963109:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.963112:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963115:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541279:541279:256:4294967295] 192.168.202.148@tcp LPNI seq info [541244:541244:8:4294967295] 00000400:00000200:0.0:1788573680.963120:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963127:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.963132:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963136:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573680.963141:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573680.963144:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573680.963164:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963166:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573680.963169:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.963173:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.963176:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573680.963178:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573680.963181:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.963186:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963195:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.963198:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963206:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963208:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.963217:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.963238:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573680.963243:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573680.963246:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308819993 00000100:00000001:0.0:1788573680.963248:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963251:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308819993 00000100:00000001:0.0:1788573680.963253:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963255:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.963258:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963260:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.963265:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.963275:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.963276:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.963278:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573680.963281:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573680.963283:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573680.963285:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.963288:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.963298:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963305:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.963306:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963308:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.963309:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.963313:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963322:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.963327:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963340:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.963343:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963344:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.963346:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963348:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.963350:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.963353:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.963357:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573680.963359:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.963364:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.963372:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151680:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.963377:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.963380:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963387:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.963389:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.963391:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.963393:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.963395:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.963398:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151808:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.963402:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.963404:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.963406:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.963408:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.963412:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573680.963414:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.963424:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573680.963426:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573680.963430:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573680.963434:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810151808, portal 10 00000100:00000001:0.0:1788573680.963436:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.963438:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573680.963444:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963451:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.963453:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573680.963456:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573680.963459:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810151808, offset 0 00000400:00000200:0.0:1788573680.963463:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963468:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963472:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573680.963475:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.963478:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573680.963481:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.963483:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963486:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541280:541280:256:4294967295] 192.168.202.148@tcp LPNI seq info [541245:541245:8:4294967295] 00000400:00000200:0.0:1788573680.963491:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963498:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.963503:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963507:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573680.963511:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573680.963514:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573680.963525:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963527:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573680.963531:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.963534:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.963538:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573680.963547:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573680.963550:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.963554:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963563:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.963566:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963573:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963575:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.963578:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.963582:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573680.963585:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573680.963587:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308819997 00000100:00000001:0.0:1788573680.963590:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963592:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308819997 00000100:00000001:0.0:1788573680.963595:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963597:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.963599:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963601:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.963605:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.963614:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.963616:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.963618:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573680.963620:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573680.963622:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573680.963624:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.963628:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.963660:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963676:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.963678:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963680:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.963681:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.963686:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963693:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.963697:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963705:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.963707:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963709:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.963711:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.963713:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.963715:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.963718:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.963721:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573680.963723:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.963727:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.963735:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151808:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.963739:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.963743:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963750:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.963752:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.963760:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.963762:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.963764:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.963768:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151936:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.963771:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.963773:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.963775:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.963777:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.963781:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573680.963783:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.963787:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573680.963790:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573680.963794:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573680.963798:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810151936, portal 4 00000100:00000001:0.0:1788573680.963800:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.963801:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573680.963808:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963814:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.963817:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573680.963819:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573680.963821:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810151936, offset 0 00000400:00000200:0.0:1788573680.963825:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963834:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573680.963837:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.963839:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573680.963842:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.963844:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573680.963848:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541281:541281:256:4294967295] 192.168.202.148@tcp LPNI seq info [541246:541246:8:4294967295] 00000400:00000200:0.0:1788573680.963852:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.963859:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.963870:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963873:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573680.963878:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573680.963881:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573680.963893:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.963895:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573680.963898:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.963901:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.963905:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573680.963907:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573680.963910:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.963914:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963922:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.963926:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.963950:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963952:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.963956:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.963959:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573680.963963:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573680.963965:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820130 00000100:00000001:0.0:1788573680.963968:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963970:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820130 00000100:00000001:0.0:1788573680.963972:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.963974:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.963976:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.963978:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.963983:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.963997:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.963999:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.964001:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573680.964004:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573680.964005:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573680.964008:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.964012:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.964020:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964028:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.964029:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964031:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.964033:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.964036:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964044:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.964048:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964055:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.964058:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964059:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.964061:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964063:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.964066:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.964069:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.964072:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573680.964074:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.964086:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.964094:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810151936:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.964098:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964101:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.964104:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810151680/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964111:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.964113:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.964115:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573680.964118:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.964119:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.964121:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573680.964123:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.964125:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573680.964128:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.964129:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.964131:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573680.964133:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.964134:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964136:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.964140:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810151808/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964148:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.964149:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.964151:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573680.964153:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.964155:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.964156:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573680.964159:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.964160:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573680.964163:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.964169:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.964171:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573680.964174:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.964175:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964177:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.964182:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810151936/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964189:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.964190:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.964192:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573680.964195:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.964196:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.964198:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573680.964200:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.964201:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573680.964204:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.964205:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.964207:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573680.964209:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.964210:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964212:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964216:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.964217:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.964220:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.964221:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.964223:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.964227:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964235:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.964236:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.964239:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.964241:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.964243:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.964246:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152064:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.964250:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.964257:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.964259:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.964261:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.964264:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573680.964267:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.964271:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573680.964273:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573680.964278:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573680.964281:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152064, portal 4 00000100:00000001:0.0:1788573680.964283:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.964285:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573680.964291:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964298:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.964300:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573680.964303:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573680.964305:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810152064, offset 0 00000400:00000200:0.0:1788573680.964309:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573680.964314:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.964317:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573680.964320:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.964323:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573680.964326:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.964328:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573680.964332:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541282:541282:256:4294967295] 192.168.202.148@tcp LPNI seq info [541247:541247:8:4294967295] 00000400:00000200:0.0:1788573680.964337:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573680.964344:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.964349:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.964352:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573680.964356:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573680.964359:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573680.964371:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573680.964379:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573680.964382:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.964386:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.964390:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573680.964392:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573680.964395:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.964399:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964408:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.964411:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964419:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.964420:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.964424:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.964427:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573680.964430:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573680.964432:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820139 00000100:00000001:0.0:1788573680.964435:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.964437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820139 00000100:00000001:0.0:1788573680.964439:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308823774 00000100:00000001:0.0:1788573680.964441:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.964443:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.964445:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.964448:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.964456:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.964458:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.964460:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573680.964463:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573680.964464:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573680.964471:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.964475:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.964484:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964490:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.964492:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964494:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.964495:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.964499:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964506:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.964510:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964516:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.964519:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964521:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.964522:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964524:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.964525:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.964528:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.964532:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573680.964534:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.964539:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.964547:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152064:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.964551:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964553:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.964562:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810152064/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.964569:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.964571:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.964573:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573680.964575:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.964577:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.964578:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573680.964580:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.964582:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573680.964585:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.964586:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.964587:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573680.964590:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.964592:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964594:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.964597:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.964599:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.964601:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.964602:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573681.537756:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573681.537763:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573681.537767:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573681.537769:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573681.537770:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573681.537771:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573681.537774:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573681.537775:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573681.537777:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573681.537780:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573681.537781:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573681.537792:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573681.537796:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573681.537801:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573681.537804:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573681.537809:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573681.537815:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788573681.537876:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573681.537882:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573681.537885:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573681.537893:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788573681.537898:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573681.537902:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788573681.537907:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573681.537910:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168b29 to hash 00000020:00000001:0.0:1788573681.537913:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573681.537927:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573681.537930:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573681.537934:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573681.537938:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573681.537943:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573681.537954:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573681.537967:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573681.537974:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573681.537976:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573681.537984:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573681.537987:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788573681.537990:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.537992:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531986816 : -109714177564800 : ffff9c372d6b5f80) 00000100:00000001:0.0:1788573681.537996:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573681.538000:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.538002:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573681.538003:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.538012:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538029:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573681.538032:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538037:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573681.538040:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538044:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573681.538046:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573681.538047:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.538049:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788573681.538051:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573681.538053:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573681.538055:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.538058:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.538059:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.538062:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.538067:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810152192:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573681.538072:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.538075:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573681.538077:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573681.538080:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.538083:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538091:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573681.538094:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573681.538100:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573681.538102:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573681.538105:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573681.538113:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573681.538115:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573681.538117:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788573681.538120:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573681.538123:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788573681.538127:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538142:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573681.538146:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538150:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573681.538153:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538156:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573681.538158:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573681.538159:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538163:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573681.538164:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573681.538166:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573681.538168:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573681.538172:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573681.538174:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573681.538175:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538187:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573681.538188:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573681.538189:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573681.538190:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788573681.538191:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573681.538193:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788573681.538195:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573681.538196:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573681.538200:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573681.538202:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573681.538205:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573681.538207:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573681.538209:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573681.538211:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573681.538212:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573681.538215:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573681.538216:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573681.538217:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573681.538218:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573681.538220:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168b29 from hash 00010000:00000001:0.0:1788573681.538222:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573681.538222:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573681.538223:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573681.538224:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573681.538225:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573681.538226:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573681.538228:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573681.538229:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573681.538230:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573681.538248:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573681.538251:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573681.538255:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573681.538256:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573681.538257:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573681.538259:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573681.538260:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573681.538260:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573681.538261:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573681.538263:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168b29 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573681.538267:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788573681.538275:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573681.538276:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573681.538278:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.538281:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5f80 x1875440810152192/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538286:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573681.538288:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.538289:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573681.538290:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.538293:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788573681.538296:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.538298:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.538299:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5f80. 00000100:00000001:0.0:1788573681.538303:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.538304:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573681.538305:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573681.538307:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573681.538310:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000001:0.0:1788573681.538533:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573681.538538:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573681.538543:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.538558:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538578:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.538581:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.538587:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573681.538591:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573681.538595:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.538602:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810152320:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573681.538609:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573681.538614:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573681.538617:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.538622:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573681.538626:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573681.538629:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.538672:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573681.538677:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573681.538686:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573681.538694:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152320, portal 25 00000100:00000001:0.0:1788573681.538698:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.538700:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573681.538710:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538727:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573681.538738:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573681.538742:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573681.538746:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810152320, offset 0 00000400:00000200:0.0:1788573681.538753:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573681.538770:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.538782:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573681.538787:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573681.538793:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573681.538802:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573681.538807:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573681.538822:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541283:541283:256:4294967295] 192.168.202.148@tcp LPNI seq info [541248:541248:8:4294967295] 00000400:00000200:0.0:1788573681.538829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.538842:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573681.538853:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.538859:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573681.538868:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573681.538873:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573681.538896:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.538899:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573681.538929:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573681.538936:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573681.538943:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573681.538946:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573681.538952:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573681.538957:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.538983:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.538991:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539014:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539020:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573681.539029:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.539036:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573681.539041:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308824351 00000100:00000001:0.0:1788573681.539049:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308823774 00000100:00000001:0.0:1788573681.539057:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308824351 00000100:00000001:0.0:1788573681.539062:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308823774 00000100:00000001:0.0:1788573681.539067:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573681.539072:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539082:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573681.539090:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573681/real 1788573681] req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573681.539107:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573681.539109:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573681.539113:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573681.539117:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573681.539120:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573681.539125:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573681.539132:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573681.539154:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539172:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573681.539175:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539178:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539180:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573681.539190:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539207:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.539215:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539224:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573681.539230:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539236:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539239:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539246:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573681.539250:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573681.539258:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573681.539268:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573681.539286:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573681.539296:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.539310:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810152320:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573681.539322:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539327:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.539333:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810152320/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539349:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573681.539352:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573681.539356:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573681.539359:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573681.539360:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.539362:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573681.539366:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.539369:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788573681.539373:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.539375:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.539378:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573681.539384:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.539386:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539389:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539395:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573681.539399:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539404:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573681.539406:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573681.539409:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.539420:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539427:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.539428:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539430:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573681.539432:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573681.539439:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.539442:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152448:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573681.539444:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573681.539445:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573681.539447:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539448:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573681.539450:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573681.539452:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.539455:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573681.539457:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573681.539459:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573681.539462:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152448, portal 10 00000100:00000001:0.0:1788573681.539463:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.539464:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573681.539468:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539473:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573681.539475:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573681.539476:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573681.539477:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810152448, offset 0 00000400:00000200:0.0:1788573681.539480:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539483:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539485:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573681.539488:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573681.539489:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573681.539491:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573681.539493:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539495:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541284:541284:256:4294967295] 192.168.202.148@tcp LPNI seq info [541249:541249:8:4294967295] 00000400:00000200:0.0:1788573681.539497:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539502:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573681.539506:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539511:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573681.539514:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573681.539516:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573681.539523:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539524:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573681.539527:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573681.539529:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573681.539532:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573681.539533:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573681.539534:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573681.539537:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539543:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.539545:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539549:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539550:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573681.539552:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573681.539555:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573681.539557:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573681.539558:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820569 00000100:00000001:0.0:1788573681.539560:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.539561:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820569 00000100:00000001:0.0:1788573681.539563:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.539565:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573681.539566:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539567:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573681.539569:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573681/real 1788573681] req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573681.539584:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573681.539584:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573681.539586:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573681.539587:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573681.539588:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573681.539590:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573681.539592:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573681.539598:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539602:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573681.539603:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539604:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539605:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573681.539607:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539611:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.539614:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539618:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573681.539619:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539620:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539621:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539622:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573681.539623:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573681.539625:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573681.539627:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573681.539628:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573681.539651:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.539658:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152448:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573681.539661:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.539663:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539667:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.539668:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539670:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573681.539672:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573681.539673:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.539675:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152576:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573681.539677:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573681.539678:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573681.539679:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539681:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573681.539683:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573681.539684:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.539687:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573681.539688:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573681.539691:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573681.539693:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152576, portal 10 00000100:00000001:0.0:1788573681.539694:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.539695:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573681.539699:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539703:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573681.539705:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573681.539706:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573681.539708:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810152576, offset 0 00000400:00000200:0.0:1788573681.539714:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539717:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539720:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573681.539722:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573681.539724:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573681.539725:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573681.539727:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539729:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541285:541285:256:4294967295] 192.168.202.148@tcp LPNI seq info [541250:541250:8:4294967295] 00000400:00000200:0.0:1788573681.539732:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539737:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573681.539740:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539742:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573681.539745:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573681.539747:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573681.539754:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539755:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573681.539757:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573681.539759:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573681.539761:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573681.539763:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573681.539764:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573681.539767:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539772:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.539774:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539778:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539779:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573681.539781:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573681.539783:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573681.539785:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573681.539790:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820574 00000100:00000001:0.0:1788573681.539792:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.539794:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820574 00000100:00000001:0.0:1788573681.539795:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.539796:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573681.539798:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.539799:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573681.539801:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573681/real 1788573681] req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573681.539807:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573681.539808:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573681.539809:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573681.539810:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573681.539811:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573681.539812:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573681.539814:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573681.539820:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539824:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573681.539825:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539826:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539827:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573681.539829:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539833:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.539835:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539839:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573681.539843:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539845:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.539845:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.539847:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573681.539848:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573681.539850:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573681.539852:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573681.539853:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573681.539856:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.539860:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152576:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573681.539863:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.539865:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539869:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.539870:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539871:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573681.539873:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573681.539874:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.539876:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152704:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573681.539877:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573681.539878:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573681.539879:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.539881:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573681.539883:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573681.539884:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.539886:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573681.539888:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573681.539890:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573681.539892:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152704, portal 4 00000100:00000001:0.0:1788573681.539896:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.539897:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573681.539902:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.539906:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573681.539907:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573681.539909:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573681.539911:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810152704, offset 0 00000400:00000200:0.0:1788573681.539913:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539929:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539932:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573681.539934:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573681.539935:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573681.539937:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573681.539938:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573681.539941:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541286:541286:256:4294967295] 192.168.202.148@tcp LPNI seq info [541251:541251:8:4294967295] 00000400:00000200:0.0:1788573681.539943:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.539948:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573681.539950:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539953:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573681.539955:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573681.539957:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573681.539967:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.539969:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573681.539972:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573681.539975:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573681.539978:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573681.539980:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573681.539982:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573681.539986:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540007:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540011:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540018:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540019:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573681.540022:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573681.540025:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573681.540027:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573681.540028:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820706 00000100:00000001:0.0:1788573681.540030:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.540032:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820706 00000100:00000001:0.0:1788573681.540033:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.540035:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573681.540036:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540037:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573681.540040:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573681/real 1788573681] req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573681.540044:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573681.540045:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573681.540046:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573681.540048:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573681.540049:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573681.540050:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573681.540052:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573681.540057:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540061:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573681.540067:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.540069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573681.540072:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540077:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.540079:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540083:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573681.540085:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540086:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.540087:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540088:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573681.540089:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573681.540091:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573681.540093:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573681.540094:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573681.540096:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.540101:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152704:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573681.540103:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540104:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540106:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810152448/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540111:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573681.540112:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573681.540113:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788573681.540115:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573681.540116:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.540120:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573681.540121:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.540122:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788573681.540124:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.540125:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.540126:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573681.540128:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.540129:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540130:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540133:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810152576/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540137:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573681.540138:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573681.540138:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788573681.540140:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573681.540140:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.540141:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573681.540142:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.540143:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573681.540144:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.540145:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.540146:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573681.540147:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.540148:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540149:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540151:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810152704/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540155:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573681.540155:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573681.540156:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573681.540157:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573681.540158:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.540159:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573681.540160:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.540161:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573681.540162:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.540165:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.540166:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573681.540168:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.540169:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540170:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540173:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573681.540174:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540175:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573681.540176:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573681.540177:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573681.540179:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540184:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.540185:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.540186:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573681.540188:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573681.540189:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.540191:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152832:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573681.540192:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573681.540194:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573681.540195:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573681.540196:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573681.540197:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573681.540199:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573681.540201:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573681.540202:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573681.540204:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573681.540207:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810152832, portal 4 00000100:00000001:0.0:1788573681.540208:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573681.540209:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788573681.540213:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540219:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573681.540221:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573681.540227:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573681.540230:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810152832, offset 0 00000400:00000200:0.0:1788573681.540235:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573681.540239:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.540243:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573681.540246:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573681.540249:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573681.540251:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573681.540254:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573681.540257:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541287:541287:256:4294967295] 192.168.202.148@tcp LPNI seq info [541252:541252:8:4294967295] 00000400:00000200:0.0:1788573681.540262:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573681.540268:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573681.540273:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.540277:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573681.540281:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573681.540284:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573681.540293:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573681.540294:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573681.540297:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573681.540299:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573681.540301:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573681.540303:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573681.540304:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573681.540307:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540313:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540315:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573706 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540319:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540320:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573681.540328:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573681.540330:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573681.540332:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573681.540334:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308820715 00000100:00000001:0.0:1788573681.540335:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.540337:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308820715 00000100:00000001:0.0:1788573681.540339:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308824351 00000100:00000001:0.0:1788573681.540340:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573681.540342:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540343:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573681.540345:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573681/real 1788573681] req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573681.540350:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573681.540351:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573681.540352:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573681.540354:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573681.540355:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573681.540356:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573681.540358:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573681.540363:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540367:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573681.540368:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540370:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.540371:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573681.540373:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540377:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573681.540379:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540387:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573681.540388:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540389:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573681.540390:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540391:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573681.540393:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573681.540394:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573681.540396:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573681.540397:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573681.540400:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573681.540405:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810152832:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573681.540408:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540409:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573681.540411:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810152832/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573706 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573681.540415:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573681.540416:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573681.540417:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788573681.540418:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573681.540419:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573681.540420:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573681.540421:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573681.540422:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573681.540423:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573681.540424:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573681.540425:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573681.540426:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573681.540427:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540431:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.540434:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573681.540434:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.540436:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573681.540437:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573681.857811:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573681.857815:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573681.857818:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573681.921967:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573685.377718:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308824351, at 4308824384 00000100:00000010:0.0:1788573685.377735:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00080000:0.0:1788573686.593739:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573686.593748:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573686.593751:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573686.593754:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573686.593758:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573686.593765:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14153 00000400:00000200:0.0:1788573686.593773:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573686.593778:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573686.593781:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573686.593784:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593796:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.593798:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593800:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573686.593803:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573686.593804:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573686.593806:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573686.593809:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573686.593811:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573686.593813:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573686.593815:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593816:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.593835:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593836:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573686.593839:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573686.593842:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593845:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573686.593847:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573686.593849:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573686.593852:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.593853:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573686.593855:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573686.593860:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573686.593863:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573686.593867:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573686.593868:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.593871:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573686.593873:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573686.593877:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573686.593883:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573686.593889:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.593896:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810152960/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573686.593904:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573686.593907:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.593911:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810152960/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573686.593928:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573686.593931:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573686.593933:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573686.593935:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573686.593936:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573686.593938:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573686.593945:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14153 00000400:00000200:0.0:1788573686.593949:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573686.593952:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573686.593957:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573686.593962:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593965:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.593970:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593972:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573686.593976:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573686.593978:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573686.593981:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573686.593984:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573686.593989:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573686.593994:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573686.593997:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.593999:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594000:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594001:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573686.594002:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573686.594008:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594010:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573686.594013:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573686.594019:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573686.594023:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594029:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573686.594033:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573686.594039:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594041:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573686.594046:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573686.594049:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573686.594052:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573686.594059:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594063:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573686.594066:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573686.594069:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573686.594074:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573686.594078:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.594083:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810153088/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573686.594093:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573686.594095:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.594099:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810153088/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573686.594110:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573686.594113:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573686.594116:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573686.594118:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573686.594121:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573686.594126:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573686.594130:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14153 00000400:00000200:0.0:1788573686.594137:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573686.594140:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573686.594143:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573686.594149:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594152:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594156:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594160:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573686.594163:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573686.594173:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573686.594176:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573686.594177:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573686.594179:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573686.594189:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573686.594195:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594198:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594201:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594204:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573686.594207:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573686.594212:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594215:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573686.594216:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573686.594220:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573686.594225:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594229:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573686.594231:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573686.594235:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594237:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573686.594239:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573686.594240:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573686.594242:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573686.594243:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594244:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573686.594247:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573686.594250:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573686.594254:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573686.594258:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.594263:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810153216/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573686.594276:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573686.594279:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.594282:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810153216/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573686.594294:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573686.594305:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573686.594311:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573686.594315:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573686.594318:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573686.594324:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573686.594329:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14153 00000400:00000200:0.0:1788573686.594338:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573686.594345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573686.594349:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573686.594353:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594358:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594363:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594367:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573686.594370:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573686.594372:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573686.594375:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573686.594379:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573686.594382:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573686.594386:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573686.594391:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594395:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594399:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594403:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573686.594415:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573686.594421:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594426:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573686.594429:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573686.594433:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573686.594436:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594439:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573686.594447:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573686.594451:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594455:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573686.594458:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573686.594462:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573686.594464:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573686.594467:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594469:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573686.594471:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573686.594475:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573686.594481:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573686.594485:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.594494:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810153344/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573686.594512:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573686.594515:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.594519:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810153344/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573686.594542:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573686.594544:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573686.594550:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573686.594553:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573686.594556:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573686.594560:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573686.594565:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14153 00000400:00000200:0.0:1788573686.594571:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573686.594577:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573686.594581:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573686.594587:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594591:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594596:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594606:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573686.594611:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573686.594613:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573686.594617:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573686.594620:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573686.594624:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573686.594628:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573686.594652:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594654:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573686.594656:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594657:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573686.594660:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573686.594663:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573686.594665:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573686.594666:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573686.594669:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573686.594673:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594678:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573686.594681:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573686.594688:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594694:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573686.594700:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573686.594705:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573686.594710:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573686.594713:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.594716:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573686.594719:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573686.594722:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573686.594725:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573686.594729:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.594737:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810153472/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573686.594771:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573686.594775:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.594797:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810153472/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573686.594830:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573686.594835:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14163) 00000020:00000001:0.0:1788573686.977838:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573686.977842:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573686.977845:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573690.433767:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308829406, at 4308829440 00000100:00000010:0.0:1788573690.433794:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788573692.097878:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573692.097884:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573692.097888:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573694.786507:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573694.786513:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573694.786516:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573694.786532:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573694] req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573694.786546:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573694.786548:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573694.786552:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37840. 00000400:00000200:0.0:1788573694.786556:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1440 00000400:00000010:0.0:1788573694.786559:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1440. 00000100:00000001:0.0:1788573694.786562:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573694.786567:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573694.786576:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786598:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573694.786609:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786613:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786616:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573694.786622:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786630:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573694.786681:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786692:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573694.786696:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786704:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786706:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786709:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573694.786713:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573694.786718:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573694.786724:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573694.786729:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573694.786735:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573694.786745:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810153984:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573694.786751:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573694.786755:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573694] req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573694.786765:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573694.786766:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573694.786769:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc379a0. 00000400:00000200:0.0:1788573694.786773:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1330 00000400:00000010:0.0:1788573694.786775:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1330. 00000100:00000001:0.0:1788573694.786784:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573694.786789:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573694.786798:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786805:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573694.786807:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786809:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786810:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573694.786815:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786823:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573694.786827:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786835:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573694.786838:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786840:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786841:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786843:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573694.786845:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573694.786848:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573694.786853:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573694.786855:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573694.786860:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573694.786869:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154112:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573694.786873:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573694.786884:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573694] req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573694.786901:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573694.786903:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573694.786905:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37f78. 00000400:00000200:0.0:1788573694.786908:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1a18 00000400:00000010:0.0:1788573694.786911:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1a18. 00000100:00000001:0.0:1788573694.786913:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573694.786917:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573694.786926:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786934:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573694.786935:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786938:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786939:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573694.786943:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786951:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573694.786956:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.786964:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573694.786966:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786968:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.786969:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.786971:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573694.786973:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573694.786975:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573694.786980:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573694.786988:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573694.786994:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573694.787003:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810153728:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573694.787008:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573694.787013:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573694] req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573694.787023:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573694.787024:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573694.787027:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37688. 00000400:00000200:0.0:1788573694.787030:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1990 00000400:00000010:0.0:1788573694.787032:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1990. 00000100:00000001:0.0:1788573694.787034:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573694.787038:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573694.787046:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787053:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573694.787055:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787057:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.787059:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573694.787062:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787070:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573694.787074:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787081:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573694.787090:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787092:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.787093:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787095:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573694.787097:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573694.787100:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573694.787104:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573694.787107:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573694.787111:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573694.787120:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810153856:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573694.787124:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573694.787129:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573694] req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573694.787138:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573694.787140:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573694.787142:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c371fc37dc0. 00000400:00000200:0.0:1788573694.787145:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788573694.787147:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788573694.787150:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573694.787153:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573694.787162:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787169:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573694.787171:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787173:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.787174:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573694.787179:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787192:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573694.787197:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787204:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573694.787206:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787208:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573694.787210:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787211:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573694.787213:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573694.787216:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573694.787220:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573694.787243:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573694.787249:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573694.787257:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810153600:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573694.787262:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787265:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573694.787268:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71180 x1875440810153984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787276:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573694.787279:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573694.787284:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573694.787289:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573694.787293:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573694.787295:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573694.787298:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573694.787301:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788573694.787305:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573694.787313:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573694.787316:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71180. 00000100:00000001:0.0:1788573694.787320:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573694.787321:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787323:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573694.787328:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810154112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787335:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573694.787336:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573694.787338:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573694.787340:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573694.787342:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573694.787343:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573694.787345:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573694.787347:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788573694.787348:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573694.787350:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573694.787351:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573694.787353:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573694.787355:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787356:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573694.787359:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810153728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787365:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573694.787367:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573694.787368:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573694.787371:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573694.787373:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573694.787374:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573694.787376:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573694.787378:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573694.787380:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573694.787381:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573694.787382:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788573694.787385:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573694.787386:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787396:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573694.787401:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810153856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787408:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573694.787410:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573694.787412:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372da37c00. 02000000:00000001:0.0:1788573694.787416:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573694.787418:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573694.787419:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573694.787421:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573694.787423:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573694.787426:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573694.787427:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573694.787429:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788573694.787431:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573694.787433:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787434:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573694.787438:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810153600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573694.787447:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573694.787448:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573694.787450:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573694.787453:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573694.787455:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573694.787456:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573694.787458:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573694.787460:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573694.787463:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573694.787464:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573694.787465:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573694.787468:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573694.787469:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787472:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573694.787481:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573694.787484:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573694.787487:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573694.787494:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573695.553741:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308834526, at 4308834560 00000100:00000010:0.0:1788573695.553758:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000100:00000001:0.0:1788573696.833976:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.833982:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.833988:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.834004:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834024:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.834028:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.834035:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.834039:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.834047:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.834057:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810154240:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573696.834066:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.834072:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.834074:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.834080:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.834088:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.834092:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.834103:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573696.834109:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573696.834121:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573696.834132:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154240, portal 25 00000100:00000001:0.0:1788573696.834138:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.834143:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573696.834154:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834167:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.834173:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573696.834178:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573696.834183:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810154240, offset 0 00000400:00000200:0.0:1788573696.834195:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573696.834216:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.834227:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573696.834237:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.834244:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573696.834251:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.834258:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573696.834269:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541298:541298:256:4294967295] 192.168.202.148@tcp LPNI seq info [541263:541263:8:4294967295] 00000400:00000200:0.0:1788573696.834280:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.834294:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.834307:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.834318:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573696.834331:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573696.834339:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573696.834374:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.834380:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573696.834390:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.834399:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.834409:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573696.834414:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573696.834421:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.834429:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834448:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.834454:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834469:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.834474:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.834481:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.834490:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573696.834496:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308839646 00000100:00000010:0.0:1788573696.834505:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573696.834521:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573696.834550:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308839646 00000100:00000010:0.0:1788573696.834556:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626a80. 00000100:00000001:0.0:1788573696.834560:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573696.834564:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.834567:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.834570:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.834576:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.834587:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.834589:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.834591:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573696.834594:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573696.834596:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573696.834599:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.834603:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.834612:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834620:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.834622:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834625:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.834627:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.834632:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834682:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.834688:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834696:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.834699:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834708:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.834709:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834712:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.834715:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.834718:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.834723:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573696.834728:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.834732:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.834741:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810154240:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573696.834746:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834749:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.834753:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810154240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834760:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.834762:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.834765:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.834768:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.834771:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.834774:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573696.834778:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.834783:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788573696.834791:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.834794:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.834798:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573696.834805:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.834809:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834814:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.834825:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.834830:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.834835:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.834839:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.834844:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.834858:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834872:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.834875:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.834880:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.834883:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.834889:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.834897:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154368:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.834906:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.834910:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.834914:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.834919:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.834926:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.834932:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.834943:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573696.834949:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573696.834956:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573696.834962:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154368, portal 10 00000100:00000001:0.0:1788573696.834967:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.834971:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573696.834984:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.834998:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.835003:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573696.835009:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573696.835015:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810154368, offset 0 00000400:00000200:0.0:1788573696.835025:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835034:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.835043:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573696.835052:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.835058:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573696.835064:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.835077:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835081:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541299:541299:256:4294967295] 192.168.202.148@tcp LPNI seq info [541264:541264:8:4294967295] 00000400:00000200:0.0:1788573696.835097:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.835116:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.835129:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.835136:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573696.835148:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573696.835155:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573696.835186:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.835191:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573696.835199:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.835208:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.835213:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573696.835218:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573696.835225:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.835233:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835249:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.835255:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835271:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.835273:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.835276:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.835283:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573696.835292:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573696.835298:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308835865 00000100:00000001:0.0:1788573696.835306:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.835310:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308835865 00000100:00000001:0.0:1788573696.835317:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.835333:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.835341:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.835346:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.835356:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.835369:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.835371:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.835373:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573696.835376:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573696.835378:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573696.835381:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.835384:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.835392:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835398:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.835400:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835402:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.835403:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.835407:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835414:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.835417:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835424:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.835426:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835428:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.835429:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835431:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.835432:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.835441:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.835444:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573696.835447:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.835452:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.835460:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154368:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.835464:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.835468:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835474:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.835476:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.835478:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.835479:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.835482:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.835485:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154496:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.835488:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.835490:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.835492:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.835494:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.835497:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573696.835500:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.835504:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573696.835507:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573696.835511:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573696.835513:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154496, portal 10 00000100:00000001:0.0:1788573696.835515:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.835516:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573696.835520:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835550:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.835553:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573696.835556:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573696.835559:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810154496, offset 0 00000400:00000200:0.0:1788573696.835563:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835568:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.835573:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573696.835577:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.835580:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573696.835583:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.835586:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835589:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541300:541300:256:4294967295] 192.168.202.148@tcp LPNI seq info [541265:541265:8:4294967295] 00000400:00000200:0.0:1788573696.835594:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.835601:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.835605:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.835607:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573696.835612:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573696.835615:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573696.835633:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.835657:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573696.835661:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.835666:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.835669:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573696.835671:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573696.835674:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.835678:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835687:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.835691:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835710:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.835711:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.835716:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.835720:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573696.835724:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573696.835727:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308835870 00000100:00000001:0.0:1788573696.835729:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.835731:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308835870 00000100:00000001:0.0:1788573696.835734:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.835736:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.835737:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.835739:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.835745:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.835754:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.835755:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.835757:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573696.835760:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573696.835762:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573696.835764:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.835768:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.835777:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835783:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.835785:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835787:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.835788:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.835792:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835805:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.835810:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835817:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.835820:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835822:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.835823:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835825:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.835827:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.835829:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.835832:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573696.835834:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.835837:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.835842:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154496:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.835845:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835847:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.835849:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810154368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835853:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.835854:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.835856:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.835858:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.835860:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.835861:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573696.835862:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.835864:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788573696.835870:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.835871:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.835872:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573696.835875:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.835875:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835877:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.835879:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810154496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835883:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.835884:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.835885:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573696.835887:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.835887:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.835888:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573696.835889:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.835890:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573696.835891:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.835892:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.835893:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573696.835894:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.835895:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835896:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.835900:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.835900:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.835902:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.835903:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.835904:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.835906:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835910:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.835911:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.835913:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.835914:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.835915:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.835917:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154624:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.835925:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.835926:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.835927:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.835929:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.835930:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573696.835932:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.835935:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573696.835938:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573696.835942:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573696.835946:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154624, portal 4 00000100:00000001:0.0:1788573696.835948:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.835949:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573696.835957:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.835964:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.835966:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573696.835969:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573696.835971:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810154624, offset 0 00000400:00000200:0.0:1788573696.835975:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835979:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.835984:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573696.835987:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.835989:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573696.835991:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.835993:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573696.835995:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541301:541301:256:4294967295] 192.168.202.148@tcp LPNI seq info [541266:541266:8:4294967295] 00000400:00000200:0.0:1788573696.835998:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.836002:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.836005:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.836008:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573696.836011:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573696.836017:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573696.836025:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.836026:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573696.836028:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.836031:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.836034:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573696.836035:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573696.836036:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.836039:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836045:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.836047:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836051:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.836052:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.836054:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.836057:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573696.836059:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573696.836060:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308836002 00000100:00000001:0.0:1788573696.836062:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.836064:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308836002 00000100:00000001:0.0:1788573696.836065:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.836066:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.836068:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.836069:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.836071:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.836076:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.836077:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.836081:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573696.836083:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573696.836084:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573696.836086:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.836088:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.836094:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836098:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.836099:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836100:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.836101:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.836103:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836107:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.836109:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836113:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.836115:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836116:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.836117:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836118:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.836120:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.836122:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.836124:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573696.836125:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.836127:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.836132:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154624:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.836137:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.836140:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836144:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.836145:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.836147:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.836148:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.836149:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.836151:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154752:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.836153:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.836154:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.836156:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.836157:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.836159:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.836160:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.836163:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573696.836164:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573696.836166:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573696.836168:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154752, portal 4 00000100:00000001:0.0:1788573696.836170:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.836171:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573696.836175:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836179:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.836181:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573696.836182:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573696.836183:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810154752, offset 0 00000400:00000200:0.0:1788573696.836186:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573696.836188:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.836190:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573696.836192:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.836197:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573696.836199:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.836200:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573696.836202:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541302:541302:256:4294967295] 192.168.202.148@tcp LPNI seq info [541267:541267:8:4294967295] 00000400:00000200:0.0:1788573696.836205:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573696.836209:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.836212:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.836214:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573696.836216:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573696.836218:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573696.836226:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573696.836227:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573696.836229:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.836232:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.836234:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573696.836235:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573696.836236:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.836239:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836244:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.836246:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836250:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.836251:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.836253:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.836255:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573696.836257:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573696.836258:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308836011 00000100:00000001:0.0:1788573696.836259:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.836260:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308836011 00000100:00000001:0.0:1788573696.836265:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308839646 00000100:00000001:0.0:1788573696.836267:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.836268:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.836269:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.836272:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.836277:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.836278:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.836279:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573696.836281:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573696.836282:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573696.836283:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.836285:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.836290:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836294:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.836295:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836297:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.836298:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.836300:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836304:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.836306:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836311:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.836313:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836314:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.836315:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836321:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.836323:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.836326:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.836330:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573696.836332:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.836336:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.836344:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810154752:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.836348:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836350:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.836354:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810154624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836360:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.836362:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.836364:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573696.836367:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.836368:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.836370:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573696.836372:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.836374:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573696.836376:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.836377:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.836379:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573696.836382:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.836383:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836385:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.836388:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810154752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.836393:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.836394:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.836395:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573696.836402:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.836403:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.836404:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573696.836405:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.836407:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573696.836408:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.836409:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.836410:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573696.836412:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.836413:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836414:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.836417:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.836418:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.836419:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.836420:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573697.217798:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573697.217803:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573697.217807:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573700.673747:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308839646, at 4308839680 00000100:00000010:0.0:1788573700.673765:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573701.953994:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.953999:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.954004:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.954017:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954034:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.954038:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.954042:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.954045:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.954050:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.954057:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810154880:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573701.954076:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.954082:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.954087:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.954092:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.954106:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573701.954111:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.954122:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573701.954129:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573701.954139:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573701.954150:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810154880, portal 25 00000100:00000001:0.0:1788573701.954156:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.954159:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287359104 : -109714422192512 : ffff9c371ed6a680) 00000100:00000040:0.0:1788573701.954171:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954192:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.954197:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573701.954200:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573701.954206:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810154880, offset 0 00000400:00000200:0.0:1788573701.954213:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573701.954222:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.954230:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573701.954237:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.954242:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573701.954246:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.954252:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573701.954257:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541303:541303:256:4294967295] 192.168.202.148@tcp LPNI seq info [541268:541268:8:4294967295] 00000400:00000200:0.0:1788573701.954265:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.954279:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.954290:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.954297:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573701.954308:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573701.954312:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573701.954334:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.954338:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573701.954344:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573701.954358:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573701.954364:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573701.954367:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573701.954372:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573701.954378:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954391:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.954397:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954423:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.954426:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573701.954433:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.954440:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573701.954446:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308844766 00000100:00000010:0.0:1788573701.954450:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573701.954458:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573701.954467:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308844766 00000100:00000010:0.0:1788573701.954472:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573701.954477:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573701.954481:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.954486:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.954488:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573701.954493:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573701] req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573701.954500:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573701.954502:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573701.954505:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573701.954510:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573701.954512:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573701.954516:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573701.954521:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573701.954564:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954585:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573701.954590:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954594:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.954597:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573701.954605:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954620:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573701.954624:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954633:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573701.954657:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954658:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.954659:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954665:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573701.954668:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573701.954674:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573701.954683:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573701.954689:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573701.954695:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573701.954707:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810154880:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573701.954715:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954720:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.954727:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a680 x1875440810154880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954777:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573701.954781:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573701.954785:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573701.954790:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573701.954793:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573701.954796:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573701.954800:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573701.954804:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573701.954809:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573701.954812:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573701.954817:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a680. 00000100:00000001:0.0:1788573701.954822:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573701.954825:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954828:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.954835:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.954837:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.954841:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.954844:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.954846:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.954853:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954870:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.954872:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.954877:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.954881:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.954886:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.954893:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155008:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.954898:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.954901:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.954904:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.954908:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.954911:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573701.954916:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.954925:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573701.954936:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573701.954942:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573701.954948:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810155008, portal 10 00000100:00000001:0.0:1788573701.954952:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.954954:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351040 : -109714422200576 : ffff9c371ed68700) 00000100:00000040:0.0:1788573701.954961:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.954972:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.954979:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573701.954983:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573701.954986:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810155008, offset 0 00000400:00000200:0.0:1788573701.954997:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955008:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955017:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573701.955024:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.955028:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573701.955033:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.955035:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955038:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541304:541304:256:4294967295] 192.168.202.148@tcp LPNI seq info [541269:541269:8:4294967295] 00000400:00000200:0.0:1788573701.955041:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955048:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.955052:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955055:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573701.955058:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573701.955060:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573701.955068:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955069:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573701.955073:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573701.955077:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573701.955080:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573701.955082:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573701.955091:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573701.955096:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955105:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955108:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955116:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955118:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573701.955122:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.955126:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573701.955129:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573701.955131:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308840985 00000100:00000001:0.0:1788573701.955134:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955137:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308840985 00000100:00000001:0.0:1788573701.955139:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955142:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.955144:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955147:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573701.955151:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573701] req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573701.955161:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573701.955162:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573701.955165:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573701.955167:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573701.955169:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573701.955170:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573701.955173:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573701.955178:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955193:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573701.955194:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955195:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955196:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573701.955199:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955204:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573701.955206:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955210:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573701.955212:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955213:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955214:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955215:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573701.955216:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573701.955218:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573701.955220:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573701.955222:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573701.955224:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573701.955229:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155008:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.955232:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.955234:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955237:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.955239:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955243:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.955244:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.955246:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.955248:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155136:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.955250:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.955252:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.955253:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955254:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955256:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573701.955258:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.955260:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573701.955262:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573701.955264:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573701.955267:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810155136, portal 10 00000100:00000001:0.0:1788573701.955268:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.955269:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287361792 : -109714422189824 : ffff9c371ed6b100) 00000100:00000040:0.0:1788573701.955273:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955278:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.955279:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573701.955281:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573701.955282:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810155136, offset 0 00000400:00000200:0.0:1788573701.955285:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955287:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955290:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573701.955292:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.955294:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573701.955296:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.955297:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955299:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541305:541305:256:4294967295] 192.168.202.148@tcp LPNI seq info [541270:541270:8:4294967295] 00000400:00000200:0.0:1788573701.955302:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955306:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.955313:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955315:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573701.955318:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573701.955320:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573701.955327:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955329:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573701.955331:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573701.955333:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573701.955335:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573701.955336:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573701.955338:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573701.955340:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955346:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955348:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955352:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955353:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573701.955355:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.955358:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573701.955360:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573701.955361:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308840989 00000100:00000001:0.0:1788573701.955362:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955364:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308840989 00000100:00000001:0.0:1788573701.955365:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955367:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.955368:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955369:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573701.955371:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573701] req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573701.955380:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573701.955381:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573701.955382:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573701.955384:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573701.955385:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573701.955387:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573701.955389:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573701.955394:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955398:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573701.955399:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955400:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955401:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573701.955417:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955421:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573701.955424:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955428:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573701.955430:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955431:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955432:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955433:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573701.955435:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573701.955436:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573701.955438:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573701.955439:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573701.955446:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573701.955451:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155136:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.955454:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.955456:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955460:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.955461:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955462:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.955464:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.955465:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.955467:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155264:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.955469:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.955470:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.955471:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955472:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955475:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573701.955476:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.955479:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573701.955480:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573701.955483:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573701.955485:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810155264, portal 4 00000100:00000001:0.0:1788573701.955486:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.955487:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573701.955491:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955496:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.955498:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573701.955499:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573701.955500:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810155264, offset 0 00000400:00000200:0.0:1788573701.955506:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955509:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955511:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573701.955514:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.955515:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573701.955517:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.955519:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955521:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541306:541306:256:4294967295] 192.168.202.148@tcp LPNI seq info [541271:541271:8:4294967295] 00000400:00000200:0.0:1788573701.955524:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955528:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.955531:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955533:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573701.955537:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573701.955540:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573701.955550:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955552:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573701.955555:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573701.955558:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573701.955562:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573701.955564:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573701.955566:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573701.955570:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955579:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955582:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955590:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955591:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573701.955593:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.955596:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573701.955598:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573701.955604:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308841121 00000100:00000001:0.0:1788573701.955606:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955607:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308841121 00000100:00000001:0.0:1788573701.955609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955610:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.955612:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955613:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573701.955616:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573701] req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573701.955621:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573701.955622:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573701.955623:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573701.955625:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573701.955626:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573701.955627:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573701.955629:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573701.955650:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955655:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573701.955656:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955657:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955658:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573701.955661:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955665:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573701.955668:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955672:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573701.955678:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955679:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955680:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955682:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573701.955683:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573701.955685:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573701.955688:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573701.955689:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573701.955692:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573701.955697:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155264:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.955699:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955701:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955702:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68700 x1875440810155008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955706:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573701.955707:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955709:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573701.955711:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573701.955712:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573701.955714:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573701.955716:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573701.955718:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573701.955721:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573701.955722:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573701.955724:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68700. 00000100:00000001:0.0:1788573701.955726:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573701.955728:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955729:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955733:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b100 x1875440810155136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955752:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573701.955753:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955755:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573701.955758:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573701.955759:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573701.955761:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573701.955763:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573701.955764:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573701.955767:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573701.955768:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573701.955769:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b100. 00000100:00000001:0.0:1788573701.955771:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573701.955772:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955773:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955776:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810155264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955780:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573701.955781:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955782:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573701.955784:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573701.955784:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573701.955785:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573701.955786:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573701.955787:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573701.955788:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573701.955789:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573701.955790:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573701.955791:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573701.955792:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955793:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955796:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.955796:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955798:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.955799:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.955800:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.955806:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955810:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.955811:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955813:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.955814:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.955816:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.955817:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155392:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.955820:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.955821:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.955822:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.955823:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.955825:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573701.955827:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.955829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573701.955831:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573701.955833:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573701.955835:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810155392, portal 4 00000100:00000001:0.0:1788573701.955837:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.955838:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573701.955842:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955846:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.955848:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573701.955849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573701.955851:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810155392, offset 0 00000400:00000200:0.0:1788573701.955853:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955856:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955858:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573701.955860:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.955862:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573701.955864:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.955865:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573701.955871:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541307:541307:256:4294967295] 192.168.202.148@tcp LPNI seq info [541272:541272:8:4294967295] 00000400:00000200:0.0:1788573701.955874:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573701.955878:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.955882:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955884:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573701.955887:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573701.955889:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573701.955896:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573701.955897:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573701.955899:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573701.955901:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573701.955904:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573701.955905:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573701.955906:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573701.955909:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955914:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.955916:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955920:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955921:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573701.955923:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.955925:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573701.955927:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573701.955928:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308841131 00000100:00000001:0.0:1788573701.955930:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955931:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308841131 00000100:00000001:0.0:1788573701.955933:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308844766 00000100:00000001:0.0:1788573701.955934:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.955938:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.955940:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573701.955942:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573701] req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573701.955948:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573701.955949:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573701.955950:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573701.955952:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573701.955953:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573701.955954:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573701.955956:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573701.955960:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955964:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573701.955965:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.955967:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.955968:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573701.955970:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955974:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573701.955977:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.955993:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573701.955999:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.956001:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573701.956002:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.956004:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573701.956006:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573701.956009:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573701.956018:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573701.956021:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573701.956026:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573701.956034:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810155392:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.956037:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.956039:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573701.956041:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810155392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.956045:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573701.956046:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573701.956047:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573701.956049:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573701.956050:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573701.956051:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573701.956052:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573701.956053:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573701.956055:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573701.956056:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573701.956057:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573701.956058:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573701.956059:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.956061:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.956063:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.956064:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.956066:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.956067:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573702.337837:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573702.337840:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573702.337843:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573703.682232:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573703.682238:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573703.682249:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573703.682253:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573703.682257:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573703.682262:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14173 00000400:00000200:0.0:1788573703.682270:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573703.682277:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573703.682281:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573703.682285:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682288:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682291:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682294:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573703.682297:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573703.682299:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573703.682301:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573703.682304:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573703.682306:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573703.682309:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573703.682312:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682315:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682317:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682319:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573703.682322:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573703.682325:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682328:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573703.682331:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573703.682334:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573703.682337:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682339:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573703.682342:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573703.682348:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573703.682351:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573703.682396:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573703.682398:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682401:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573703.682403:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573703.682406:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573703.682411:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573703.682415:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.682423:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810155648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573703.682432:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573703.682435:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.682441:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810155648/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573703.682467:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573703.682470:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573703.682474:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573703.682475:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573703.682477:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573703.682480:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573703.682483:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14173 00000400:00000200:0.0:1788573703.682487:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573703.682492:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573703.682495:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573703.682499:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682501:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682504:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682507:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573703.682512:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573703.682516:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573703.682521:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573703.682525:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573703.682535:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573703.682541:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573703.682551:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682558:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682565:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682570:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573703.682575:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573703.682582:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682587:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573703.682591:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573703.682599:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573703.682605:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682610:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573703.682617:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573703.682624:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682628:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573703.682671:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573703.682676:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573703.682683:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573703.682685:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682689:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573703.682692:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573703.682696:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573703.682705:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573703.682712:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.682726:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810155776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573703.682742:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573703.682748:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.682757:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810155776/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573703.682800:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573703.682807:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573703.682815:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573703.682821:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573703.682824:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573703.682830:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573703.682838:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14173 00000400:00000200:0.0:1788573703.682850:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573703.682856:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573703.682860:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573703.682868:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682874:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682879:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682883:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573703.682890:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573703.682894:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573703.682898:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573703.682901:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573703.682906:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573703.682913:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573703.682921:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682927:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.682933:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682940:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573703.682947:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573703.682955:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573703.682962:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573703.682975:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573703.682980:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573703.682983:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.682986:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573703.682991:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573703.682998:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683004:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573703.683010:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573703.683018:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573703.683025:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573703.683027:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683029:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573703.683032:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573703.683036:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573703.683044:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573703.683050:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.683062:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810155904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573703.683079:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573703.683084:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.683091:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810155904/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573703.683097:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573703.683099:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573703.683102:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573703.683103:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573703.683106:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573703.683110:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573703.683115:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14173 00000400:00000200:0.0:1788573703.683122:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573703.683128:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573703.683132:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573703.683142:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683146:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.683151:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683156:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573703.683160:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573703.683162:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573703.683166:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573703.683172:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573703.683178:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573703.683185:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573703.683193:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683197:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.683203:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683209:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573703.683215:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573703.683223:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683226:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573703.683230:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573703.683236:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573703.683241:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683246:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573703.683251:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573703.683257:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683261:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573703.683268:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573703.683272:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573703.683277:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573703.683281:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683286:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573703.683290:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573703.683293:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573703.683300:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573703.683311:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.683318:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810156032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573703.683328:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573703.683331:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.683338:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810156032/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573703.683356:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573703.683378:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573703.683384:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573703.683388:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573703.683392:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573703.683399:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573703.683404:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14173 00000400:00000200:0.0:1788573703.683412:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573703.683419:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573703.683423:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573703.683429:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683434:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.683439:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683444:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573703.683449:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573703.683452:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573703.683454:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573703.683457:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573703.683462:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573703.683467:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573703.683472:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683477:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573703.683488:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683492:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573703.683496:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573703.683502:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573703.683506:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573703.683510:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573703.683515:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573703.683518:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683523:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573703.683527:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573703.683532:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683537:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573703.683543:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573703.683547:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573703.683550:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573703.683552:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.683556:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573703.683559:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573703.683562:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573703.683566:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573703.683571:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.683579:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810156160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573703.683586:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573703.683590:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.683593:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810156160/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573703.683609:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573703.683615:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14180) 00000020:00000001:0.0:1788573707.457794:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573707.457798:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573707.457801:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573707.521730:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308846495, at 4308846528 00000100:00000010:0.0:1788573707.521749:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000020:00000001:0.0:1788573712.577828:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573712.577832:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573712.577835:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573712.961836:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308851934, at 4308851968 00000100:00000010:0.0:1788573712.961848:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb1e0. 00000020:00000001:0.0:1788573717.697878:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573717.697884:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573717.697888:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573718.081772:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308857054, at 4308857088 00000100:00000010:0.0:1788573718.081790:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000100:00000001:0.0:1788573719.362027:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573719.362032:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573719.362048:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573719.362066:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362086:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.362092:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.362098:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573719.362103:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573719.362109:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.362116:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810157568:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573719.362123:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573719.362127:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573719.362131:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.362135:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573719.362142:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573719.362143:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.362150:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573719.362155:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573719.362163:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573719.362172:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810157568, portal 25 00000100:00000001:0.0:1788573719.362183:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573719.362186:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573719.362196:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362212:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573719.362217:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573719.362222:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573719.362226:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810157568, offset 0 00000400:00000200:0.0:1788573719.362236:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573719.362246:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.362253:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573719.362260:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573719.362268:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573719.362274:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573719.362279:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573719.362286:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541323:541323:256:4294967295] 192.168.202.148@tcp LPNI seq info [541288:541288:8:4294967295] 00000400:00000200:0.0:1788573719.362293:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.362312:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573719.362323:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.362331:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573719.362341:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573719.362348:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 35 secs later) 00000800:00000100:0.0:1788573719.362371:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.362376:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573719.362382:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573719.362389:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573719.362396:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573719.362399:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573719.362404:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573719.362411:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362447:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.362454:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362484:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.362488:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573719.362496:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.362506:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573719.362513:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308862174 00000100:00000010:0.0:1788573719.362519:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573719.362541:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573719.362551:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308862174 00000100:00000010:0.0:1788573719.362560:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116261e0. 00000100:00000001:0.0:1788573719.362566:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573719.362574:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573719.362578:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.362583:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573719.362590:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573719/real 1788573719] req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573719.362623:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573719.362627:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573719.362632:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573719.362705:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573719.362711:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573719.362717:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573719.362725:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573719.362746:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362760:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573719.362771:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362777:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.362782:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573719.362797:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362822:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573719.362830:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362836:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573719.362838:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362840:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.362841:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362843:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573719.362845:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573719.362847:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573719.362851:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573719.362861:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573719.362864:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573719.362870:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810157568:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573719.362874:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362875:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.362878:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810157568/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362883:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573719.362886:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573719.362888:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573719.362890:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573719.362892:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573719.362898:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573719.362900:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573719.362903:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34800. 02000000:00000001:0.0:1788573719.362906:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573719.362908:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573719.362910:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573719.362913:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573719.362914:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362916:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.362922:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573719.362924:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.362926:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573719.362927:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573719.362928:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573719.362931:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.362936:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.362937:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.362939:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573719.362940:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573719.362941:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.362943:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157696:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573719.362956:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573719.362957:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573719.362958:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.362960:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573719.362962:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573719.362963:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.362966:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573719.362968:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573719.362971:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573719.362973:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810157696, portal 10 00000100:00000001:0.0:1788573719.362975:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573719.362976:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573719.362981:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363016:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573719.363018:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573719.363020:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573719.363022:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810157696, offset 0 00000400:00000200:0.0:1788573719.363025:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363029:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363032:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573719.363034:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573719.363047:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573719.363049:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573719.363050:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363053:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541324:541324:256:4294967295] 192.168.202.148@tcp LPNI seq info [541289:541289:8:4294967295] 00000400:00000200:0.0:1788573719.363056:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363063:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573719.363068:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363071:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573719.363075:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573719.363078:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 35 secs later) 00000800:00000100:0.0:1788573719.363090:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363092:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573719.363095:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573719.363099:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573719.363103:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573719.363105:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573719.363107:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573719.363112:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363121:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363134:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363135:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573719.363139:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573719.363142:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573719.363144:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573719.363146:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308858393 00000100:00000001:0.0:1788573719.363148:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363150:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308858393 00000100:00000001:0.0:1788573719.363151:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363153:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573719.363154:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363156:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573719.363158:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573719/real 1788573719] req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573719.363164:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573719.363165:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573719.363166:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573719.363167:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573719.363168:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573719.363170:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573719.363172:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573719.363177:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363181:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573719.363182:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363183:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363184:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573719.363186:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363194:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573719.363197:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363202:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573719.363203:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363204:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363205:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363206:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573719.363207:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573719.363209:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573719.363211:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573719.363212:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573719.363215:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573719.363220:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157696:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573719.363222:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573719.363224:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363228:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.363229:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363231:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573719.363232:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573719.363233:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.363235:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157824:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573719.363237:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573719.363238:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573719.363242:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363244:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573719.363246:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573719.363247:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.363249:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573719.363251:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573719.363254:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573719.363256:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810157824, portal 10 00000100:00000001:0.0:1788573719.363257:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573719.363258:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573719.363262:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363267:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573719.363268:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573719.363270:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573719.363272:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810157824, offset 0 00000400:00000200:0.0:1788573719.363274:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363276:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363279:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573719.363281:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573719.363283:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573719.363284:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573719.363286:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363288:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541325:541325:256:4294967295] 192.168.202.148@tcp LPNI seq info [541290:541290:8:4294967295] 00000400:00000200:0.0:1788573719.363291:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363295:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573719.363298:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363300:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573719.363303:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573719.363305:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 35 secs later) 00000800:00000100:0.0:1788573719.363312:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363316:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573719.363318:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573719.363321:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573719.363323:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573719.363324:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573719.363326:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573719.363328:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363334:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363336:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363340:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363341:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573719.363343:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573719.363346:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573719.363348:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573719.363349:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308858397 00000100:00000001:0.0:1788573719.363351:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363352:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308858397 00000100:00000001:0.0:1788573719.363354:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363355:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573719.363356:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363358:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573719.363360:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573719/real 1788573719] req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573719.363364:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573719.363365:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573719.363366:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573719.363367:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573719.363368:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573719.363373:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573719.363375:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573719.363380:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363384:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573719.363385:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363386:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363387:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573719.363389:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363393:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573719.363396:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363400:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573719.363402:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363403:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363403:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363405:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573719.363406:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573719.363407:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573719.363409:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573719.363410:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573719.363413:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573719.363417:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157824:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573719.363419:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573719.363425:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363430:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.363431:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363432:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573719.363434:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573719.363435:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.363437:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157952:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573719.363439:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573719.363440:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573719.363441:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363443:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573719.363445:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573719.363446:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.363448:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573719.363449:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573719.363452:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573719.363454:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810157952, portal 4 00000100:00000001:0.0:1788573719.363466:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573719.363467:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573719.363472:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363479:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573719.363482:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573719.363484:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573719.363486:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810157952, offset 0 00000400:00000200:0.0:1788573719.363491:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363495:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363499:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573719.363502:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573719.363504:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573719.363507:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573719.363510:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363518:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541326:541326:256:4294967295] 192.168.202.148@tcp LPNI seq info [541291:541291:8:4294967295] 00000400:00000200:0.0:1788573719.363535:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363541:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573719.363546:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363550:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573719.363554:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573719.363557:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 35 secs later) 00000800:00000100:0.0:1788573719.363568:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363570:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573719.363573:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573719.363577:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573719.363579:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573719.363580:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573719.363582:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573719.363585:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363602:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363604:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363608:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363610:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573719.363612:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573719.363614:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573719.363616:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573719.363617:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308858529 00000100:00000001:0.0:1788573719.363619:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363620:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308858529 00000100:00000001:0.0:1788573719.363622:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363623:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573719.363629:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363630:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573719.363633:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573719/real 1788573719] req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573719.363666:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573719.363667:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573719.363668:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573719.363670:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573719.363671:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573719.363673:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573719.363675:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573719.363680:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363685:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573719.363686:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363687:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363688:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573719.363690:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363694:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573719.363697:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363701:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573719.363703:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363704:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363705:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363706:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573719.363707:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573719.363709:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573719.363715:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573719.363716:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573719.363719:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573719.363724:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810157952:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573719.363727:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573719.363729:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363733:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.363733:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363735:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573719.363736:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573719.363737:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.363739:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158080:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573719.363741:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573719.363742:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573719.363744:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573719.363745:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573719.363747:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573719.363748:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573719.363751:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573719.363752:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573719.363755:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573719.363757:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158080, portal 4 00000100:00000001:0.0:1788573719.363758:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573719.363759:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573719.363763:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363767:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573719.363772:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573719.363773:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573719.363775:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810158080, offset 0 00000400:00000200:0.0:1788573719.363778:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363780:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363783:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573719.363785:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573719.363787:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573719.363789:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573719.363790:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573719.363792:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541327:541327:256:4294967295] 192.168.202.148@tcp LPNI seq info [541292:541292:8:4294967295] 00000400:00000200:0.0:1788573719.363795:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573719.363799:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573719.363801:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363804:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573719.363806:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573719.363808:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 35 secs later) 00000800:00000100:0.0:1788573719.363818:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573719.363820:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573719.363822:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573719.363825:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573719.363827:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573719.363828:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573719.363830:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573719.363832:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363837:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363839:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363843:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363844:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573719.363849:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573719.363851:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573719.363853:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573719.363855:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308858539 00000100:00000001:0.0:1788573719.363856:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363858:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308858539 00000100:00000001:0.0:1788573719.363860:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308862174 00000100:00000001:0.0:1788573719.363861:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573719.363862:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.363863:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573719.363866:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573719/real 1788573719] req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573719.363870:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573719.363871:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573719.363872:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573719.363874:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573719.363875:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573719.363876:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573719.363878:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573719.363883:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363887:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573719.363888:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363889:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363890:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573719.363892:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363896:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573719.363898:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363907:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573719.363908:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363909:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573719.363910:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363912:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573719.363913:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573719.363914:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573719.363916:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573719.363917:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573719.363920:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573719.363925:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158080:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573719.363938:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363940:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363942:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810157696/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363946:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573719.363947:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573719.363948:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573719.363949:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573719.363950:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573719.363951:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573719.363952:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573719.363953:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36c00. 02000000:00000001:0.0:1788573719.363955:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573719.363956:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573719.363957:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573719.363959:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573719.363960:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363964:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363967:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810157824/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.363971:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573719.363972:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573719.363973:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573719.363974:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573719.363975:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573719.363976:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573719.363977:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573719.363978:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36000. 02000000:00000001:0.0:1788573719.363979:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573719.363980:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573719.363980:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573719.363982:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573719.363982:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.363983:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.363985:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810157952/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.364003:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573719.364004:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573719.364006:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573719.364008:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573719.364009:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573719.364009:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573719.364011:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573719.364011:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36800. 02000000:00000001:0.0:1788573719.364013:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573719.364013:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573719.364014:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573719.364015:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573719.364016:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.364017:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573719.364020:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810158080/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573744 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573719.364031:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573719.364032:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573719.364033:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573719.364035:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573719.364036:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573719.364036:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573719.364037:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573719.364038:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34400. 02000000:00000001:0.0:1788573719.364039:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573719.364040:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573719.364041:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573719.364042:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573719.364043:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.364050:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.364052:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573719.364053:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.364054:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573719.364055:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573719.809943:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573722.817803:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573722.817809:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573722.817813:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573723.201725:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308862174, at 4308862208 00000100:00000010:0.0:1788573723.201741:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573724.482000:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573724.482006:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573724.482009:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573724.482018:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482045:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.482049:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.482057:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573724.482062:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573724.482070:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.482085:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810158208:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573724.482090:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573724.482094:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573724.482099:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.482104:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573724.482111:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573724.482114:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.482123:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573724.482128:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573724.482136:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573724.482145:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158208, portal 25 00000100:00000001:0.0:1788573724.482148:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573724.482151:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573724.482159:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482169:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573724.482172:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573724.482175:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573724.482180:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810158208, offset 0 00000400:00000200:0.0:1788573724.482194:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573724.482204:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.482214:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573724.482226:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573724.482233:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573724.482249:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573724.482256:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573724.482262:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541328:541328:256:4294967295] 192.168.202.148@tcp LPNI seq info [541293:541293:8:4294967295] 00000400:00000200:0.0:1788573724.482269:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.482285:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573724.482301:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.482312:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573724.482333:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573724.482341:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788573724.482371:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.482376:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573724.482382:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573724.482388:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573724.482395:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573724.482400:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573724.482404:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573724.482411:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482429:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.482435:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482449:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.482453:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573724.482460:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.482468:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573724.482473:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308867294 00000100:00000010:0.0:1788573724.482480:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573724.482488:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573724.482494:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308867294 00000100:00000010:0.0:1788573724.482498:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116261e0. 00000100:00000001:0.0:1788573724.482503:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573724.482509:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573724.482515:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.482520:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573724.482527:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573724/real 1788573724] req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573724.482539:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573724.482547:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573724.482551:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573724.482555:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573724.482559:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573724.482563:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573724.482570:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573724.482587:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482602:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573724.482606:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482612:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.482616:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573724.482623:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482675:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573724.482684:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482694:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573724.482699:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482704:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.482707:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482712:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573724.482714:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573724.482718:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573724.482725:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573724.482729:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573724.482734:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573724.482741:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810158208:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573724.482754:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482758:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.482765:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810158208/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482780:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573724.482784:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573724.482787:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573724.482791:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573724.482794:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573724.482797:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573724.482801:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573724.482804:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788573724.482810:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573724.482813:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573724.482817:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573724.482820:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573724.482823:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482827:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.482835:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573724.482838:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.482841:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573724.482844:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573724.482847:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573724.482853:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482868:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.482869:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.482870:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573724.482873:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573724.482876:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.482880:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158336:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573724.482885:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573724.482902:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573724.482905:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.482916:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573724.482920:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573724.482922:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.482933:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573724.482938:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573724.482942:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573724.482947:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158336, portal 10 00000100:00000001:0.0:1788573724.482952:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573724.482954:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573724.482965:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.482985:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573724.482991:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573724.482995:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573724.482998:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810158336, offset 0 00000400:00000200:0.0:1788573724.483006:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483009:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483013:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573724.483016:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573724.483018:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573724.483021:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573724.483024:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483028:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541329:541329:256:4294967295] 192.168.202.148@tcp LPNI seq info [541294:541294:8:4294967295] 00000400:00000200:0.0:1788573724.483033:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483039:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573724.483044:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483048:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573724.483053:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573724.483056:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788573724.483068:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483070:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573724.483077:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573724.483080:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573724.483083:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573724.483084:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573724.483086:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573724.483089:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483095:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483097:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483102:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483103:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573724.483105:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573724.483108:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573724.483110:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573724.483112:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308863513 00000100:00000001:0.0:1788573724.483114:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483116:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308863513 00000100:00000001:0.0:1788573724.483117:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483118:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573724.483120:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483121:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573724.483124:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573724/real 1788573724] req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573724.483129:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573724.483129:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573724.483131:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573724.483132:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573724.483133:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573724.483134:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573724.483139:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573724.483145:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483149:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573724.483150:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483151:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483152:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573724.483154:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483158:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573724.483161:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483166:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573724.483168:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483169:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483171:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483172:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573724.483174:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573724.483177:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573724.483181:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573724.483183:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573724.483187:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573724.483195:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158336:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573724.483199:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573724.483203:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483223:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.483225:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483227:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573724.483229:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573724.483231:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.483235:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158464:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573724.483238:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573724.483240:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573724.483241:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483243:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483246:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573724.483248:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.483252:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573724.483254:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573724.483258:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573724.483261:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158464, portal 10 00000100:00000001:0.0:1788573724.483263:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573724.483265:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573724.483270:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483277:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573724.483279:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573724.483281:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573724.483284:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810158464, offset 0 00000400:00000200:0.0:1788573724.483288:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483292:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483296:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573724.483299:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573724.483302:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573724.483304:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573724.483307:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483315:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541330:541330:256:4294967295] 192.168.202.148@tcp LPNI seq info [541295:541295:8:4294967295] 00000400:00000200:0.0:1788573724.483318:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483323:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573724.483326:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483329:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573724.483332:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573724.483334:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788573724.483341:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483343:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573724.483345:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573724.483347:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573724.483349:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573724.483350:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573724.483352:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573724.483355:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483361:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483363:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483367:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483368:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573724.483371:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573724.483373:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573724.483375:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573724.483376:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308863517 00000100:00000001:0.0:1788573724.483378:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483380:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308863517 00000100:00000001:0.0:1788573724.483382:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483383:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573724.483387:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483389:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573724.483391:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573724/real 1788573724] req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573724.483397:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573724.483398:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573724.483400:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573724.483401:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573724.483402:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573724.483403:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573724.483405:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573724.483410:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483415:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573724.483416:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483417:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483418:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573724.483420:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483424:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573724.483426:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483430:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573724.483432:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483433:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483433:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483435:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573724.483436:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573724.483437:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573724.483442:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573724.483444:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573724.483447:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573724.483452:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158464:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573724.483454:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573724.483456:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483460:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.483462:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483463:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573724.483465:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573724.483466:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.483468:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158592:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573724.483469:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573724.483471:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573724.483472:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483473:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483475:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573724.483476:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.483479:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573724.483481:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573724.483483:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573724.483485:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158592, portal 4 00000100:00000001:0.0:1788573724.483486:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573724.483487:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573724.483491:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483495:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573724.483501:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573724.483502:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573724.483504:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810158592, offset 0 00000400:00000200:0.0:1788573724.483506:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483509:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483512:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573724.483514:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573724.483515:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573724.483517:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573724.483519:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483521:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541331:541331:256:4294967295] 192.168.202.148@tcp LPNI seq info [541296:541296:8:4294967295] 00000400:00000200:0.0:1788573724.483524:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483528:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573724.483531:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483533:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573724.483536:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573724.483538:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788573724.483545:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483546:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573724.483548:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573724.483550:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573724.483552:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573724.483553:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573724.483555:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573724.483557:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483562:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483564:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483569:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483570:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573724.483575:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573724.483577:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573724.483580:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573724.483581:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308863649 00000100:00000001:0.0:1788573724.483583:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483584:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308863649 00000100:00000001:0.0:1788573724.483586:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483587:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573724.483588:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483590:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573724.483592:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573724/real 1788573724] req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573724.483597:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573724.483597:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573724.483599:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573724.483600:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573724.483602:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573724.483604:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573724.483607:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573724.483617:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483623:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573724.483625:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483627:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483628:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573724.483632:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483667:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573724.483671:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483681:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573724.483683:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483685:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483685:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483687:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573724.483688:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573724.483690:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573724.483693:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573724.483694:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573724.483697:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573724.483702:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158592:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573724.483704:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483706:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483708:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810158336/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483712:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573724.483713:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483714:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573724.483716:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573724.483717:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573724.483718:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573724.483720:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573724.483721:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788573724.483723:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573724.483724:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573724.483725:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573724.483727:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573724.483728:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483732:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483734:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810158464/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483738:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573724.483739:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483740:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573724.483742:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573724.483742:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573724.483743:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573724.483744:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573724.483745:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d000. 02000000:00000001:0.0:1788573724.483746:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573724.483747:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573724.483748:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573724.483749:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573724.483750:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483751:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483753:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810158592/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483756:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573724.483757:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483758:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573724.483759:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573724.483760:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573724.483761:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573724.483762:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573724.483762:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2ec00. 02000000:00000001:0.0:1788573724.483764:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573724.483764:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573724.483765:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573724.483766:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573724.483767:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483768:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483771:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573724.483771:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483773:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573724.483776:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573724.483777:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573724.483780:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483784:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.483785:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483787:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573724.483788:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573724.483789:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.483791:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158720:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573724.483793:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573724.483795:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573724.483796:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573724.483797:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573724.483798:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573724.483800:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573724.483802:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573724.483804:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573724.483806:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573724.483809:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158720, portal 4 00000100:00000001:0.0:1788573724.483810:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573724.483811:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788573724.483815:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483819:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573724.483821:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573724.483822:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573724.483824:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810158720, offset 0 00000400:00000200:0.0:1788573724.483826:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483831:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573724.483834:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573724.483836:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573724.483843:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573724.483846:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573724.483849:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541332:541332:256:4294967295] 192.168.202.148@tcp LPNI seq info [541297:541297:8:4294967295] 00000400:00000200:0.0:1788573724.483853:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573724.483860:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573724.483865:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483869:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573724.483872:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573724.483875:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788573724.483886:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573724.483904:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573724.483907:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573724.483911:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573724.483915:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573724.483916:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573724.483918:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573724.483921:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483927:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.483929:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483933:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483934:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573724.483937:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573724.483939:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573724.483941:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573724.483943:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308863659 00000100:00000001:0.0:1788573724.483944:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483946:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308863659 00000100:00000001:0.0:1788573724.483952:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308867294 00000100:00000001:0.0:1788573724.483954:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573724.483955:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.483957:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573724.483960:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573724/real 1788573724] req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573724.483965:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573724.483966:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573724.483967:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573724.483969:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573724.483970:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573724.483971:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573724.483973:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573724.483978:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483982:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573724.483983:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.483984:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.483985:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573724.483987:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483991:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573724.483993:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.483998:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573724.483999:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.484000:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573724.484001:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.484007:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573724.484008:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573724.484010:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573724.484012:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573724.484014:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573724.484017:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573724.484023:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810158720:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573724.484027:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.484029:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573724.484032:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810158720/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573724.484038:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573724.484040:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573724.484042:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573724.484045:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573724.484046:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573724.484048:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573724.484050:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573724.484052:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f400. 02000000:00000001:0.0:1788573724.484054:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573724.484055:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573724.484057:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573724.484060:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573724.484061:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.484063:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.484067:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573724.484068:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.484070:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573724.484072:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.314385:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573725.314389:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573725.314398:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573725.314407:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314425:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.314428:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.314433:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573725.314436:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573725.314439:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.314443:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810158976:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573725.314454:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573725.314459:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573725.314464:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.314468:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573725.314474:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573725.314479:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.314489:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573725.314494:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573725.314502:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573725.314510:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810158976, portal 25 00000100:00000001:0.0:1788573725.314515:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573725.314517:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286344192 : -109714423207424 : ffff9c371ec72a00) 00000100:00000040:0.0:1788573725.314524:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314543:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573725.314554:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573725.314558:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573725.314563:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810158976, offset 0 00000400:00000200:0.0:1788573725.314573:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573725.314582:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.314591:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573725.314598:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573725.314603:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573725.314608:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573725.314619:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573725.314626:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541333:541333:256:4294967295] 192.168.202.148@tcp LPNI seq info [541298:541298:8:4294967295] 00000400:00000200:0.0:1788573725.314651:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.314664:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573725.314674:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.314682:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573725.314693:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573725.314699:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573725.314721:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.314726:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573725.314732:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573725.314742:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573725.314749:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573725.314752:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573725.314758:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573725.314765:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314778:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.314782:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314790:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.314793:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573725.314799:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.314805:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573725.314809:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308868127 00000100:00000001:0.0:1788573725.314818:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308867294 00000100:00000001:0.0:1788573725.314825:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308868127 00000100:00000001:0.0:1788573725.314830:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308867294 00000100:00000001:0.0:1788573725.314834:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573725.314845:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.314849:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573725.314858:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573725/real 1788573725] req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573725.314888:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573725.314890:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573725.314894:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573725.314898:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573725.314902:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573725.314907:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573725.314914:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573725.314937:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314945:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573725.314947:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.314950:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.314953:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573725.314959:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314973:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573725.314980:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.314991:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573725.314997:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315008:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315011:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315015:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573725.315019:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573725.315024:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573725.315030:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573725.315046:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573725.315053:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573725.315062:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810158976:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573725.315068:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315072:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315078:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810158976/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315088:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573725.315092:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315096:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573725.315100:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573725.315103:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573725.315106:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573725.315109:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573725.315112:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2dc00. 02000000:00000001:0.0:1788573725.315115:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573725.315118:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573725.315120:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788573725.315122:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573725.315123:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315126:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315133:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573725.315137:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315141:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573725.315144:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573725.315147:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573725.315153:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315168:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.315172:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315178:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573725.315182:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573725.315195:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.315206:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159104:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573725.315214:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573725.315219:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573725.315222:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315227:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315232:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573725.315235:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.315239:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573725.315244:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573725.315250:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573725.315255:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159104, portal 10 00000100:00000001:0.0:1788573725.315259:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573725.315262:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286341504 : -109714423210112 : ffff9c371ec71f80) 00000100:00000040:0.0:1788573725.315271:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315280:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573725.315284:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573725.315287:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573725.315290:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810159104, offset 0 00000400:00000200:0.0:1788573725.315300:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315307:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315313:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573725.315319:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573725.315324:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573725.315328:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573725.315333:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315338:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541334:541334:256:4294967295] 192.168.202.148@tcp LPNI seq info [541299:541299:8:4294967295] 00000400:00000200:0.0:1788573725.315344:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315351:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573725.315356:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315364:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573725.315368:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573725.315371:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573725.315379:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315380:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573725.315382:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573725.315385:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573725.315388:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573725.315389:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573725.315391:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573725.315393:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315399:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315401:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315406:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315407:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573725.315409:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573725.315412:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573725.315414:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573725.315416:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308864345 00000100:00000001:0.0:1788573725.315418:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315419:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308864345 00000100:00000001:0.0:1788573725.315421:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315422:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573725.315423:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315425:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573725.315427:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573725/real 1788573725] req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573725.315441:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573725.315442:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573725.315443:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573725.315445:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573725.315446:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573725.315447:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573725.315450:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573725.315455:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315459:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573725.315460:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315462:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315463:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573725.315465:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315469:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573725.315471:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315475:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573725.315477:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315478:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315479:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315480:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573725.315481:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573725.315482:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573725.315485:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573725.315486:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573725.315492:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573725.315497:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159104:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573725.315499:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573725.315501:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315505:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.315506:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315508:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573725.315509:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573725.315510:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.315512:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159232:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573725.315514:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573725.315515:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573725.315516:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315517:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315519:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573725.315520:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.315522:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573725.315524:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573725.315526:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573725.315528:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159232, portal 10 00000100:00000001:0.0:1788573725.315529:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573725.315530:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788573725.315534:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315539:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573725.315541:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573725.315542:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573725.315543:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810159232, offset 0 00000400:00000200:0.0:1788573725.315549:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315552:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315554:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573725.315556:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573725.315558:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573725.315560:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573725.315561:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315563:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541335:541335:256:4294967295] 192.168.202.148@tcp LPNI seq info [541300:541300:8:4294967295] 00000400:00000200:0.0:1788573725.315566:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315570:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573725.315573:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315575:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573725.315578:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573725.315580:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573725.315587:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315588:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573725.315590:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573725.315592:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573725.315594:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573725.315596:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573725.315597:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573725.315599:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315604:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315606:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315611:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315612:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573725.315613:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573725.315616:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573725.315618:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573725.315622:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308864349 00000100:00000001:0.0:1788573725.315624:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315625:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308864349 00000100:00000001:0.0:1788573725.315627:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315628:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573725.315629:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315630:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573725.315633:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573725/real 1788573725] req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573725.315655:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573725.315657:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573725.315658:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573725.315661:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573725.315662:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573725.315664:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573725.315667:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573725.315672:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315676:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573725.315677:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315678:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315679:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573725.315681:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315685:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573725.315688:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315692:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573725.315697:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315698:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315699:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315701:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573725.315702:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573725.315704:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573725.315706:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573725.315707:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573725.315710:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573725.315715:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159232:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573725.315717:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573725.315719:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315723:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.315724:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315726:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573725.315727:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573725.315728:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.315730:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159360:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573725.315732:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573725.315733:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573725.315735:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.315736:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315738:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573725.315739:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.315742:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573725.315744:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573725.315746:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573725.315748:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159360, portal 4 00000100:00000001:0.0:1788573725.315752:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573725.315753:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337920 : -109714423213696 : ffff9c371ec71180) 00000100:00000040:0.0:1788573725.315758:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315763:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573725.315764:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573725.315766:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573725.315768:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810159360, offset 0 00000400:00000200:0.0:1788573725.315770:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315772:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315775:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573725.315777:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573725.315778:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573725.315780:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573725.315781:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573725.315783:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541336:541336:256:4294967295] 192.168.202.148@tcp LPNI seq info [541301:541301:8:4294967295] 00000400:00000200:0.0:1788573725.315786:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.315790:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573725.315793:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315795:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573725.315798:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573725.315800:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573725.315807:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.315808:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573725.315810:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573725.315812:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573725.315814:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573725.315815:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573725.315817:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573725.315819:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315831:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315834:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315838:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315839:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573725.315841:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573725.315843:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573725.315846:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573725.315847:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308864482 00000100:00000001:0.0:1788573725.315849:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315850:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308864482 00000100:00000001:0.0:1788573725.315852:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.315853:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573725.315854:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.315855:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573725.315858:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573725/real 1788573725] req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573725.315862:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573725.315863:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573725.315864:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573725.315865:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573725.315867:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573725.315868:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573725.315870:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573725.315889:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315894:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573725.315900:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315902:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315903:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573725.315905:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315910:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573725.315912:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315917:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573725.315918:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315919:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.315920:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315921:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573725.315923:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573725.315924:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573725.315926:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573725.315927:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573725.315930:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573725.315934:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159360:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573725.315937:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315939:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315941:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810159104/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315945:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573725.315946:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315948:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573725.315950:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573725.315951:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573725.315954:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573725.315956:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573725.315957:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f400. 02000000:00000001:0.0:1788573725.315959:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573725.315960:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573725.315962:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788573725.315964:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573725.315965:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315966:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315968:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810159232/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315972:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573725.315973:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315975:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573725.315976:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573725.315976:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573725.315977:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573725.315978:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573725.315979:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2ec00. 02000000:00000001:0.0:1788573725.315980:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573725.315981:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573725.315982:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788573725.315983:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573725.315984:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.315985:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.315986:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71180 x1875440810159360/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.315990:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573725.315991:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573725.315992:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573725.315993:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573725.315994:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573725.315994:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573725.315995:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573725.315996:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d000. 02000000:00000001:0.0:1788573725.315997:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573725.316000:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573725.316001:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71180. 00000100:00000001:0.0:1788573725.316003:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573725.316004:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316005:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316008:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573725.316009:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.316010:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573725.316011:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573725.316012:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573725.316015:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316019:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.316020:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.316022:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573725.316023:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573725.316024:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.316026:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159488:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573725.316028:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573725.316029:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573725.316030:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573725.316031:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573725.316033:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573725.316034:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573725.316037:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573725.316039:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573725.316041:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573725.316043:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159488, portal 4 00000100:00000001:0.0:1788573725.316044:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573725.316045:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286344192 : -109714423207424 : ffff9c371ec72a00) 00000100:00000040:0.0:1788573725.316050:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316054:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573725.316055:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573725.316060:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573725.316062:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810159488, offset 0 00000400:00000200:0.0:1788573725.316065:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573725.316067:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.316070:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573725.316072:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573725.316074:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573725.316076:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573725.316077:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573725.316079:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541337:541337:256:4294967295] 192.168.202.148@tcp LPNI seq info [541302:541302:8:4294967295] 00000400:00000200:0.0:1788573725.316082:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573725.316086:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573725.316089:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.316091:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573725.316094:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573725.316096:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 29 secs later) 00000800:00000100:0.0:1788573725.316104:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573725.316105:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573725.316107:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573725.316109:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573725.316111:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573725.316112:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573725.316114:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573725.316116:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316121:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.316123:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316128:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.316129:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573725.316135:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573725.316137:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573725.316139:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573725.316141:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308864491 00000100:00000001:0.0:1788573725.316142:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.316144:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308864491 00000100:00000001:0.0:1788573725.316145:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308868127 00000100:00000001:0.0:1788573725.316147:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573725.316148:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.316149:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573725.316151:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573725/real 1788573725] req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573725.316156:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573725.316157:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573725.316159:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573725.316160:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573725.316161:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573725.316162:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573725.316164:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573725.316172:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316179:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573725.316181:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316183:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.316184:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573725.316188:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316196:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573725.316200:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316214:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573725.316216:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316218:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573725.316220:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316221:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573725.316222:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573725.316224:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573725.316227:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573725.316228:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573725.316231:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573725.316237:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159488:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573725.316239:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316240:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573725.316242:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810159488/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573725.316246:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573725.316247:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573725.316249:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573725.316250:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573725.316251:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573725.316252:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573725.316254:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573725.316255:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788573725.316256:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573725.316257:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573725.316258:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788573725.316260:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573725.316261:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316266:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.316268:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573725.316269:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.316271:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573725.316272:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573727.937859:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573727.937865:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573727.937870:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573729.153719:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308868127, at 4308868160 00000100:00000010:0.0:1788573729.153730:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573730.626081:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573730.626087:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573730.626092:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573730.626104:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626118:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573730.626120:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.626124:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573730.626129:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573730.626133:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573730.626138:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810159616:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573730.626145:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573730.626150:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573730.626155:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.626160:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573730.626167:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573730.626170:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.626177:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573730.626184:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573730.626195:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573730.626205:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159616, portal 25 00000100:00000001:0.0:1788573730.626210:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573730.626212:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573730.626221:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626247:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573730.626255:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573730.626259:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573730.626262:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810159616, offset 0 00000400:00000200:0.0:1788573730.626273:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573730.626285:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.626296:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573730.626303:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573730.626311:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573730.626318:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573730.626325:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573730.626332:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541338:541338:256:4294967295] 192.168.202.148@tcp LPNI seq info [541303:541303:8:4294967295] 00000400:00000200:0.0:1788573730.626340:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.626354:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573730.626367:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.626382:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573730.626396:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573730.626404:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573730.626430:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.626435:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573730.626444:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573730.626453:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573730.626461:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573730.626465:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573730.626471:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573730.626480:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626499:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.626505:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626537:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.626543:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573730.626550:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.626558:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573730.626562:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308873438 00000100:00000010:0.0:1788573730.626572:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573730.626584:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573730.626598:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308873438 00000100:00000010:0.0:1788573730.626608:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626cc0. 00000100:00000001:0.0:1788573730.626616:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573730.626623:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573730.626630:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.626678:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573730.626687:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573730/real 1788573730] req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573730.626717:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573730.626722:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573730.626728:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573730.626735:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573730.626739:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573730.626746:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573730.626755:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573730.626780:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626790:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573730.626794:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626798:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.626801:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573730.626809:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626844:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573730.626858:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626877:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573730.626881:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626885:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.626886:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626888:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573730.626889:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573730.626891:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573730.626895:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573730.626905:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573730.626909:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573730.626915:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810159616:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573730.626920:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626922:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.626925:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810159616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626930:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573730.626933:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573730.626935:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573730.626938:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573730.626940:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573730.626942:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573730.626944:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573730.626947:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573730.626950:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573730.626956:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573730.626958:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573730.626961:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573730.626962:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626964:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.626970:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573730.626972:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.626973:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573730.626974:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573730.626976:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573730.626978:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.626984:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573730.626985:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.626987:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573730.626988:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573730.626990:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573730.626992:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159744:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573730.626994:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573730.626995:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573730.626997:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.626998:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573730.627000:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573730.627001:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.627005:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573730.627007:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573730.627010:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573730.627012:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159744, portal 10 00000100:00000001:0.0:1788573730.627014:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573730.627015:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573730.627020:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627025:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573730.627027:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573730.627032:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573730.627034:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810159744, offset 0 00000400:00000200:0.0:1788573730.627037:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627040:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627043:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573730.627046:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573730.627048:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573730.627050:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573730.627052:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627054:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541339:541339:256:4294967295] 192.168.202.148@tcp LPNI seq info [541304:541304:8:4294967295] 00000400:00000200:0.0:1788573730.627057:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627062:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573730.627065:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627067:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573730.627070:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573730.627072:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573730.627083:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627085:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573730.627087:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573730.627090:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573730.627092:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573730.627094:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573730.627096:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573730.627098:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627104:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.627106:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627111:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627112:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573730.627117:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573730.627121:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573730.627123:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573730.627125:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308869657 00000100:00000001:0.0:1788573730.627127:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627129:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308869657 00000100:00000001:0.0:1788573730.627130:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627132:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573730.627133:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627134:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573730.627137:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573730/real 1788573730] req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573730.627142:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573730.627144:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573730.627145:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573730.627147:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573730.627148:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573730.627149:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573730.627152:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573730.627158:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627162:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573730.627162:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627164:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627164:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573730.627167:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627171:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573730.627173:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627182:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573730.627184:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627186:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627187:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627188:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573730.627189:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573730.627190:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573730.627193:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573730.627194:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573730.627197:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573730.627202:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159744:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573730.627205:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573730.627207:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627211:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573730.627212:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627213:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573730.627214:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573730.627216:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573730.627217:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159872:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573730.627219:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573730.627220:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573730.627221:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627222:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573730.627225:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573730.627226:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.627231:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573730.627233:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573730.627235:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573730.627237:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810159872, portal 10 00000100:00000001:0.0:1788573730.627239:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573730.627240:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573730.627244:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627249:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573730.627250:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573730.627252:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573730.627253:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810159872, offset 0 00000400:00000200:0.0:1788573730.627256:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627259:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627261:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573730.627263:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573730.627264:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573730.627266:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573730.627268:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627270:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541340:541340:256:4294967295] 192.168.202.148@tcp LPNI seq info [541305:541305:8:4294967295] 00000400:00000200:0.0:1788573730.627273:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627277:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573730.627280:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627282:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573730.627284:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573730.627286:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573730.627293:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627294:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573730.627296:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573730.627299:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573730.627301:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573730.627305:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573730.627307:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573730.627310:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627316:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.627318:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627323:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627324:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573730.627326:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573730.627328:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573730.627330:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573730.627332:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308869661 00000100:00000001:0.0:1788573730.627334:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627335:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308869661 00000100:00000001:0.0:1788573730.627337:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627338:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573730.627339:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627340:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573730.627343:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573730/real 1788573730] req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573730.627347:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573730.627348:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573730.627350:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573730.627351:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573730.627353:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573730.627354:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573730.627356:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573730.627361:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627369:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573730.627370:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627371:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627372:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573730.627374:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627379:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573730.627381:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627386:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573730.627388:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627388:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627389:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627390:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573730.627391:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573730.627393:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573730.627395:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573730.627396:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573730.627398:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573730.627403:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810159872:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573730.627405:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573730.627407:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627412:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573730.627413:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627417:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573730.627419:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573730.627420:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573730.627423:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160000:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573730.627425:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573730.627426:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573730.627428:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627429:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573730.627431:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573730.627432:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.627435:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573730.627437:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573730.627439:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573730.627441:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160000, portal 4 00000100:00000001:0.0:1788573730.627443:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573730.627444:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573730.627448:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627453:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573730.627455:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573730.627456:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573730.627458:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810160000, offset 0 00000400:00000200:0.0:1788573730.627460:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627462:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627464:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573730.627466:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573730.627467:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573730.627469:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573730.627470:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627472:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541341:541341:256:4294967295] 192.168.202.148@tcp LPNI seq info [541306:541306:8:4294967295] 00000400:00000200:0.0:1788573730.627475:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627479:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573730.627486:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627488:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573730.627491:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573730.627493:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573730.627501:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627502:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573730.627504:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573730.627506:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573730.627508:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573730.627510:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573730.627511:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573730.627514:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627519:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.627521:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627525:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627526:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573730.627528:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573730.627530:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573730.627533:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573730.627534:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308869793 00000100:00000001:0.0:1788573730.627536:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627537:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308869793 00000100:00000001:0.0:1788573730.627539:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627540:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573730.627541:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627542:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573730.627545:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573730/real 1788573730] req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573730.627553:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573730.627554:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573730.627556:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573730.627557:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573730.627558:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573730.627560:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573730.627562:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573730.627571:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627578:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573730.627579:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627582:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627583:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573730.627587:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627595:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573730.627599:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627606:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573730.627609:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627611:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627612:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627614:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573730.627617:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573730.627620:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573730.627623:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573730.627626:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573730.627656:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573730.627662:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160000:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573730.627665:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573730.627667:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627672:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573730.627673:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627676:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573730.627678:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573730.627680:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573730.627683:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160128:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573730.627686:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573730.627688:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573730.627690:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573730.627692:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573730.627698:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573730.627700:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573730.627703:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573730.627707:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573730.627711:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573730.627714:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160128, portal 4 00000100:00000001:0.0:1788573730.627715:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573730.627717:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287362688 : -109714422188928 : ffff9c371ed6b480) 00000100:00000040:0.0:1788573730.627724:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627730:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573730.627732:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573730.627734:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573730.627736:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810160128, offset 0 00000400:00000200:0.0:1788573730.627747:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627752:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627755:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573730.627757:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573730.627760:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573730.627762:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573730.627765:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573730.627781:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541342:541342:256:4294967295] 192.168.202.148@tcp LPNI seq info [541307:541307:8:4294967295] 00000400:00000200:0.0:1788573730.627785:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573730.627791:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573730.627795:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627799:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573730.627803:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573730.627805:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 24 secs later) 00000800:00000100:0.0:1788573730.627816:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573730.627818:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573730.627821:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573730.627824:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573730.627827:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573730.627829:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573730.627831:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573730.627835:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627843:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.627846:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627851:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627853:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573730.627856:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573730.627858:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573730.627861:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573730.627869:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308869803 00000100:00000001:0.0:1788573730.627871:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627873:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308869803 00000100:00000001:0.0:1788573730.627875:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308873438 00000100:00000001:0.0:1788573730.627877:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573730.627879:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.627881:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573730.627885:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573730/real 1788573730] req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573730.627893:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573730.627895:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573730.627896:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573730.627899:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573730.627901:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573730.627903:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573730.627906:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573730.627914:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627922:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573730.627924:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627926:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627927:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573730.627931:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627939:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573730.627943:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627948:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573730.627955:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627956:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573730.627957:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627958:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573730.627959:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573730.627961:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573730.627964:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573730.627966:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573730.627968:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573730.627973:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160128:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573730.627976:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.627978:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.627980:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810159744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.627984:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573730.627985:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573730.627987:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573730.627988:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573730.627989:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573730.627991:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573730.627992:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573730.627994:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573730.627996:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573730.627997:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573730.627998:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573730.628000:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573730.628001:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.628002:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.628005:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810159872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.628015:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573730.628016:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573730.628018:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573730.628019:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573730.628020:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573730.628021:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573730.628022:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573730.628023:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37c00. 02000000:00000001:0.0:1788573730.628024:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573730.628025:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573730.628025:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573730.628027:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573730.628028:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.628029:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.628031:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810160000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.628035:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573730.628040:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573730.628041:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573730.628043:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573730.628043:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573730.628044:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573730.628045:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573730.628046:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573730.628047:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573730.628048:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573730.628049:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573730.628050:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573730.628051:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.628053:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573730.628055:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b480 x1875440810160128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573730.628059:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573730.628059:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573730.628063:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573730.628065:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573730.628066:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573730.628067:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573730.628068:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573730.628069:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573730.628071:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573730.628071:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573730.628072:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b480. 00000100:00000001:0.0:1788573730.628073:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573730.628074:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.628075:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573730.628078:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573730.628078:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573730.628080:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573730.628081:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573733.057824:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573733.057829:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573733.057833:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573734.465748:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308873438, at 4308873472 00000100:00000010:0.0:1788573734.465765:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573735.746074:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573735.746082:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573735.746088:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573735.746104:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.746144:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.746148:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.746156:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573735.746163:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573735.746168:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.746188:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810160256:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573735.746206:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573735.746215:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573735.746223:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.746243:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573735.746252:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573735.746259:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.746270:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573735.746276:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573735.746289:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573735.746301:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160256, portal 25 00000100:00000001:0.0:1788573735.746305:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573735.746308:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788573735.746319:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.746344:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573735.746351:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573735.746359:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573735.746367:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810160256, offset 0 00000400:00000200:0.0:1788573735.746379:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573735.746396:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.746416:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573735.746427:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573735.746438:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573735.746448:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573735.746453:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573735.746465:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541343:541343:256:4294967295] 192.168.202.148@tcp LPNI seq info [541308:541308:8:4294967295] 00000400:00000200:0.0:1788573735.746483:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.746494:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573735.746507:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.746523:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573735.746541:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573735.746547:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 19 secs later) 00000800:00000100:0.0:1788573735.746568:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.746572:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573735.746587:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573735.746597:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573735.746617:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573735.746625:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573735.746682:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573735.746688:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.746715:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.746727:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.746757:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.746759:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573735.746768:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.746776:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573735.746780:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308878559 00000100:00000010:0.0:1788573735.746786:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573735.746795:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573735.746811:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308878559 00000100:00000010:0.0:1788573735.746818:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626a80. 00000100:00000001:0.0:1788573735.746830:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573735.746838:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573735.746844:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.746851:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573735.746855:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573735/real 1788573735] req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573735.746874:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573735.746881:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573735.746890:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573735.746902:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573735.746907:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573735.746916:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573735.746930:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573735.746955:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.746981:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573735.746989:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.746993:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.746995:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573735.747006:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747062:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573735.747071:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747089:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573735.747099:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747109:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.747114:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747123:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573735.747127:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573735.747138:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573735.747149:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573735.747163:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573735.747176:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573735.747210:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810160256:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573735.747227:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747240:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.747249:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810160256/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747302:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573735.747307:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573735.747310:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573735.747316:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573735.747320:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573735.747328:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573735.747338:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573735.747342:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788573735.747352:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573735.747356:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573735.747358:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573735.747371:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573735.747373:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747377:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747384:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573735.747388:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.747393:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573735.747396:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573735.747400:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573735.747408:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747422:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.747424:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.747429:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573735.747433:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573735.747440:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.747445:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160384:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573735.747453:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573735.747456:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573735.747463:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.747468:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573735.747471:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573735.747475:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.747481:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573735.747491:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573735.747496:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573735.747504:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160384, portal 10 00000100:00000001:0.0:1788573735.747508:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573735.747510:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573735.747520:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747546:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573735.747548:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573735.747552:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573735.747555:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810160384, offset 0 00000400:00000200:0.0:1788573735.747567:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573735.747575:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.747584:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573735.747591:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573735.747596:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573735.747604:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573735.747610:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573735.747615:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541344:541344:256:4294967295] 192.168.202.148@tcp LPNI seq info [541309:541309:8:4294967295] 00000400:00000200:0.0:1788573735.747620:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.747631:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573735.747652:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.747655:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573735.747675:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573735.747678:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 19 secs later) 00000800:00000100:0.0:1788573735.747687:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.747689:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573735.747694:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573735.747698:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573735.747701:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573735.747702:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573735.747708:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573735.747712:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747718:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.747720:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747724:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.747726:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573735.747729:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573735.747731:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573735.747734:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573735.747735:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308874778 00000100:00000001:0.0:1788573735.747737:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.747739:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308874778 00000100:00000001:0.0:1788573735.747740:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.747742:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573735.747743:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.747745:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573735.747747:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573735/real 1788573735] req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573735.747752:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573735.747753:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573735.747754:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573735.747755:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573735.747757:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573735.747758:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573735.747760:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573735.747765:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747776:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573735.747777:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747779:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.747780:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573735.747782:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747786:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573735.747789:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747794:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573735.747795:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747796:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.747797:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747798:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573735.747799:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573735.747801:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573735.747804:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573735.747805:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573735.747807:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573735.747812:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160384:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573735.747815:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573735.747817:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747822:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.747823:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.747828:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573735.747829:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573735.747831:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.747833:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160512:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573735.747836:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573735.747837:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573735.747839:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.747840:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573735.747843:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573735.747844:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.747846:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573735.747848:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573735.747850:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573735.747852:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160512, portal 10 00000100:00000001:0.0:1788573735.747853:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573735.747854:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573735.747858:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747863:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573735.747864:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573735.747866:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573735.747867:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810160512, offset 0 00000400:00000200:0.0:1788573735.747870:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573735.747872:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.747874:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573735.747876:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573735.747878:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573735.747880:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573735.747881:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573735.747883:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541345:541345:256:4294967295] 192.168.202.148@tcp LPNI seq info [541310:541310:8:4294967295] 00000400:00000200:0.0:1788573735.747886:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.747890:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573735.747898:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.747900:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573735.747903:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573735.747905:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 19 secs later) 00000800:00000100:0.0:1788573735.747912:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.747913:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573735.747915:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573735.747918:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573735.747920:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573735.747921:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573735.747922:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573735.747925:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747930:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.747932:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747936:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.747937:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573735.747939:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573735.747941:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573735.747943:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573735.747944:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308874782 00000100:00000001:0.0:1788573735.747946:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.747947:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308874782 00000100:00000001:0.0:1788573735.747948:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.747950:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573735.747951:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.747952:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573735.747954:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573735/real 1788573735] req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573735.747963:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573735.747964:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573735.747965:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573735.747967:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573735.747968:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573735.747969:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573735.747971:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573735.747977:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747980:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573735.747981:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747982:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.747983:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573735.747985:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747989:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573735.747992:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.747996:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573735.747998:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.747999:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.748000:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748001:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573735.748002:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573735.748003:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573735.748005:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573735.748007:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573735.748012:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573735.748017:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160512:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573735.748020:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573735.748022:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748026:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.748027:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.748029:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573735.748030:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573735.748031:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.748033:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160640:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573735.748035:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573735.748036:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573735.748037:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.748038:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748040:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573735.748042:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.748044:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573735.748046:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573735.748049:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573735.748051:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160640, portal 4 00000100:00000001:0.0:1788573735.748052:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573735.748053:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573735.748057:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748061:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573735.748062:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573735.748064:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573735.748065:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810160640, offset 0 00000400:00000200:0.0:1788573735.748071:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573735.748073:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.748076:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573735.748078:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573735.748079:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573735.748081:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573735.748083:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573735.748085:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541346:541346:256:4294967295] 192.168.202.148@tcp LPNI seq info [541311:541311:8:4294967295] 00000400:00000200:0.0:1788573735.748087:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.748091:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573735.748094:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.748095:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573735.748098:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573735.748099:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 19 secs later) 00000800:00000100:0.0:1788573735.748106:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.748108:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573735.748109:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573735.748111:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573735.748113:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573735.748115:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573735.748116:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573735.748118:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748123:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748126:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748130:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748131:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573735.748133:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573735.748135:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573735.748137:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573735.748141:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308874914 00000100:00000001:0.0:1788573735.748143:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.748144:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308874914 00000100:00000001:0.0:1788573735.748146:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.748147:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573735.748148:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748149:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573735.748152:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573735/real 1788573735] req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573735.748157:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573735.748158:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573735.748159:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573735.748160:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573735.748162:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573735.748163:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573735.748165:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573735.748170:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748174:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573735.748175:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748177:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.748178:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573735.748180:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748184:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573735.748186:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748190:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573735.748194:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748195:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.748196:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748197:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573735.748199:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573735.748201:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573735.748203:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573735.748205:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573735.748207:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573735.748212:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160640:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573735.748215:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748217:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748220:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810160384/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748227:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573735.748229:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748231:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573735.748234:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573735.748235:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573735.748237:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573735.748239:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573735.748241:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573735.748244:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573735.748246:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573735.748247:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573735.748250:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573735.748251:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748253:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748257:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810160512/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748275:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573735.748277:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748279:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573735.748282:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573735.748284:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573735.748285:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573735.748287:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573735.748289:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573735.748291:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573735.748292:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573735.748294:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573735.748296:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573735.748298:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748300:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748304:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810160640/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748311:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573735.748313:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748315:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573735.748317:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573735.748318:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573735.748319:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573735.748320:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573735.748321:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573735.748323:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573735.748324:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573735.748325:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573735.748326:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573735.748327:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748329:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748332:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573735.748333:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748334:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573735.748335:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573735.748337:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573735.748346:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748350:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.748352:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.748354:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573735.748355:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573735.748356:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.748358:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160768:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573735.748360:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573735.748362:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573735.748363:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573735.748364:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748366:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573735.748368:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573735.748370:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573735.748372:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573735.748374:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573735.748377:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160768, portal 4 00000100:00000001:0.0:1788573735.748378:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573735.748379:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573735.748383:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748388:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573735.748389:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573735.748391:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573735.748392:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810160768, offset 0 00000400:00000200:0.0:1788573735.748395:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573735.748397:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.748400:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573735.748402:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573735.748403:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573735.748405:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573735.748407:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573735.748412:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541347:541347:256:4294967295] 192.168.202.148@tcp LPNI seq info [541312:541312:8:4294967295] 00000400:00000200:0.0:1788573735.748414:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573735.748419:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573735.748422:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.748424:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573735.748427:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573735.748429:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 19 secs later) 00000800:00000100:0.0:1788573735.748436:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573735.748437:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573735.748439:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573735.748442:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573735.748444:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573735.748445:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573735.748446:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573735.748449:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748454:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748456:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748461:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748462:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573735.748464:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573735.748466:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573735.748468:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573735.748469:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308874923 00000100:00000001:0.0:1788573735.748471:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.748472:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308874923 00000100:00000001:0.0:1788573735.748474:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308878559 00000100:00000001:0.0:1788573735.748476:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573735.748480:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748481:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573735.748484:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573735/real 1788573735] req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573735.748489:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573735.748490:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573735.748492:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573735.748493:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573735.748494:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573735.748495:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573735.748497:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573735.748502:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748506:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573735.748508:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748509:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.748510:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573735.748512:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748516:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573735.748518:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748522:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573735.748523:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748524:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573735.748525:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748526:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573735.748527:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573735.748529:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573735.748534:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573735.748535:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573735.748538:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573735.748543:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810160768:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573735.748545:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748546:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573735.748548:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810160768/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573735.748552:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573735.748553:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573735.748554:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573735.748556:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573735.748557:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573735.748558:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573735.748559:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573735.748560:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788573735.748562:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573735.748563:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573735.748564:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573735.748565:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573735.748566:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748567:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.748569:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573735.748570:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.748571:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573735.748573:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573738.177832:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573738.177837:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573738.177841:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573739.585753:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308878559, at 4308878592 00000100:00000010:0.0:1788573739.585765:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573740.866075:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573740.866083:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573740.866088:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573740.866098:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866118:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.866122:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.866130:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573740.866134:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573740.866140:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.866147:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810160896:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573740.866156:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573740.866163:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573740.866167:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.866173:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573740.866182:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573740.866186:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.866197:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573740.866205:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573740.866218:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573740.866229:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810160896, portal 25 00000100:00000001:0.0:1788573740.866236:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573740.866240:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360000 : -109714422191616 : ffff9c371ed6aa00) 00000100:00000040:0.0:1788573740.866253:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866274:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573740.866282:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573740.866288:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573740.866292:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810160896, offset 0 00000400:00000200:0.0:1788573740.866306:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573740.866323:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.866337:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573740.866348:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573740.866367:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573740.866375:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573740.866382:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573740.866394:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541348:541348:256:4294967295] 192.168.202.148@tcp LPNI seq info [541313:541313:8:4294967295] 00000400:00000200:0.0:1788573740.866418:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.866433:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573740.866447:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.866457:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573740.866470:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573740.866477:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573740.866521:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.866526:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573740.866537:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573740.866548:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573740.866558:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573740.866577:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573740.866585:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573740.866594:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866629:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.866677:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866698:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.866704:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573740.866714:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.866733:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573740.866739:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308883679 00000100:00000010:0.0:1788573740.866748:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573740.866760:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573740.866774:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308883679 00000100:00000010:0.0:1788573740.866790:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626e40. 00000100:00000001:0.0:1788573740.866797:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573740.866805:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573740.866809:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.866816:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573740.866826:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573740/real 1788573740] req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573740.866862:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573740.866868:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573740.866873:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573740.866880:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573740.866886:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573740.866892:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573740.866902:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573740.866926:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866947:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573740.866950:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.866954:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.866958:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573740.866966:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.866988:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573740.866994:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867010:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573740.867016:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867019:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867021:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867033:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573740.867037:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573740.867041:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573740.867049:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573740.867062:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573740.867069:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573740.867077:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810160896:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573740.867081:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867083:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.867086:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6aa00 x1875440810160896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867091:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573740.867093:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573740.867096:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573740.867099:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573740.867102:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573740.867103:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573740.867105:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573740.867107:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35400. 02000000:00000001:0.0:1788573740.867112:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573740.867113:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573740.867115:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6aa00. 00000100:00000001:0.0:1788573740.867117:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573740.867118:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867120:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867127:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573740.867128:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867130:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573740.867131:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573740.867132:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573740.867135:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867148:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.867149:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867151:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573740.867153:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573740.867154:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.867156:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161024:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573740.867158:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573740.867160:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573740.867161:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867162:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573740.867164:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573740.867166:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.867169:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573740.867172:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573740.867175:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573740.867178:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161024, portal 10 00000100:00000001:0.0:1788573740.867179:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573740.867181:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287363584 : -109714422188032 : ffff9c371ed6b800) 00000100:00000040:0.0:1788573740.867186:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867191:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573740.867193:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573740.867194:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573740.867196:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810161024, offset 0 00000400:00000200:0.0:1788573740.867199:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867202:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867204:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573740.867207:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573740.867209:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573740.867210:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573740.867212:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867217:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541349:541349:256:4294967295] 192.168.202.148@tcp LPNI seq info [541314:541314:8:4294967295] 00000400:00000200:0.0:1788573740.867221:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867226:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573740.867229:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867232:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573740.867235:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573740.867237:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573740.867245:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867247:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573740.867249:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573740.867252:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573740.867254:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573740.867256:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573740.867258:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573740.867260:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867266:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.867268:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867273:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867274:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573740.867276:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573740.867280:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573740.867282:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573740.867283:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308879897 00000100:00000001:0.0:1788573740.867285:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867287:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308879897 00000100:00000001:0.0:1788573740.867289:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867290:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573740.867297:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867298:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573740.867301:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573740/real 1788573740] req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573740.867307:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573740.867319:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573740.867321:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573740.867322:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573740.867323:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573740.867325:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573740.867327:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573740.867332:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867337:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573740.867338:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867339:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867340:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573740.867342:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867346:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573740.867348:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867352:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573740.867354:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867355:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867355:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867357:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573740.867358:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573740.867359:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573740.867365:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573740.867366:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573740.867369:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573740.867374:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161024:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573740.867377:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573740.867378:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867382:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.867383:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867384:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573740.867386:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573740.867387:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.867389:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161152:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573740.867391:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573740.867393:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573740.867394:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867395:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573740.867397:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573740.867398:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.867400:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573740.867401:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573740.867404:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573740.867406:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161152, portal 10 00000100:00000001:0.0:1788573740.867407:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573740.867408:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287358208 : -109714422193408 : ffff9c371ed6a300) 00000100:00000040:0.0:1788573740.867413:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867417:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573740.867421:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573740.867423:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573740.867425:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810161152, offset 0 00000400:00000200:0.0:1788573740.867428:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867442:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867445:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573740.867447:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573740.867449:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573740.867450:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573740.867452:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867454:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541350:541350:256:4294967295] 192.168.202.148@tcp LPNI seq info [541315:541315:8:4294967295] 00000400:00000200:0.0:1788573740.867457:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867462:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573740.867465:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867467:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573740.867469:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573740.867471:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573740.867478:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867479:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573740.867482:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573740.867485:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573740.867488:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573740.867490:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573740.867492:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573740.867496:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867505:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.867508:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867527:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867534:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573740.867539:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573740.867543:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573740.867547:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573740.867549:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308879901 00000100:00000001:0.0:1788573740.867552:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867555:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308879901 00000100:00000001:0.0:1788573740.867557:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867560:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573740.867562:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867577:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573740.867582:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573740/real 1788573740] req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573740.867592:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573740.867593:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573740.867595:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573740.867598:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573740.867600:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573740.867603:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573740.867605:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573740.867620:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867625:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573740.867625:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867627:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867628:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573740.867630:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867649:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573740.867658:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867663:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573740.867664:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867666:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867667:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867668:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573740.867669:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573740.867671:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573740.867673:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573740.867674:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573740.867677:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573740.867683:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161152:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573740.867685:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573740.867687:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867691:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.867692:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867693:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573740.867695:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573740.867696:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.867698:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161280:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573740.867700:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573740.867702:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573740.867703:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867704:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573740.867706:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573740.867708:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.867714:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573740.867716:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573740.867719:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573740.867721:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161280, portal 4 00000100:00000001:0.0:1788573740.867723:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573740.867724:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287362688 : -109714422188928 : ffff9c371ed6b480) 00000100:00000040:0.0:1788573740.867729:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867733:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573740.867735:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573740.867737:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573740.867738:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810161280, offset 0 00000400:00000200:0.0:1788573740.867741:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867743:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867746:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573740.867748:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573740.867750:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573740.867752:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573740.867753:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573740.867756:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541351:541351:256:4294967295] 192.168.202.148@tcp LPNI seq info [541316:541316:8:4294967295] 00000400:00000200:0.0:1788573740.867760:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.867766:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573740.867772:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867775:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573740.867779:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573740.867782:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573740.867798:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.867800:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573740.867803:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573740.867807:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573740.867811:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573740.867817:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573740.867821:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573740.867824:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867832:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.867835:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867842:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867844:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573740.867848:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573740.867852:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573740.867855:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573740.867857:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308880034 00000100:00000001:0.0:1788573740.867860:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867862:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308880034 00000100:00000001:0.0:1788573740.867864:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.867866:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573740.867868:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.867870:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573740.867874:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573740/real 1788573740] req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573740.867882:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573740.867883:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573740.867885:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573740.867888:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573740.867890:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573740.867891:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573740.867894:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573740.867902:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867914:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573740.867916:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867918:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867919:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573740.867923:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867930:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573740.867933:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867940:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573740.867942:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867944:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.867946:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.867947:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573740.867950:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573740.867952:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573740.867955:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573740.867957:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573740.867962:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573740.867969:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161280:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573740.867973:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573740.867976:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.867982:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.867983:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.867991:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573740.867992:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573740.867995:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.867998:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161408:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573740.868001:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573740.868003:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573740.868005:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573740.868006:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573740.868009:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573740.868011:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573740.868014:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573740.868017:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573740.868021:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573740.868023:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161408, portal 4 00000100:00000001:0.0:1788573740.868026:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573740.868027:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287350144 : -109714422201472 : ffff9c371ed68380) 00000100:00000040:0.0:1788573740.868034:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868041:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573740.868043:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573740.868045:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573740.868047:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810161408, offset 0 00000400:00000200:0.0:1788573740.868050:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573740.868053:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.868056:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573740.868058:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573740.868060:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573740.868061:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573740.868063:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573740.868065:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541352:541352:256:4294967295] 192.168.202.148@tcp LPNI seq info [541317:541317:8:4294967295] 00000400:00000200:0.0:1788573740.868068:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573740.868073:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573740.868081:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.868083:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573740.868086:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573740.868088:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 14 secs later) 00000800:00000100:0.0:1788573740.868095:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573740.868097:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573740.868099:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573740.868101:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573740.868104:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573740.868105:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573740.868107:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573740.868109:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868115:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.868117:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868122:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.868123:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573740.868125:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573740.868128:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573740.868130:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573740.868132:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308880043 00000100:00000001:0.0:1788573740.868134:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.868135:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308880043 00000100:00000001:0.0:1788573740.868137:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308883679 00000100:00000001:0.0:1788573740.868138:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573740.868139:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.868140:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573740.868143:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573740/real 1788573740] req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573740.868152:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573740.868153:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573740.868154:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573740.868156:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573740.868157:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573740.868159:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573740.868161:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573740.868167:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868171:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573740.868172:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868173:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.868174:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573740.868176:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868180:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573740.868183:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868187:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573740.868188:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868189:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573740.868190:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868191:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573740.868192:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573740.868194:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573740.868196:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573740.868197:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573740.868203:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573740.868209:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161408:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573740.868211:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868212:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.868214:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b800 x1875440810161024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868218:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573740.868219:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573740.868221:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573740.868222:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573740.868223:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573740.868224:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573740.868226:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573740.868227:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573740.868229:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573740.868230:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573740.868231:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b800. 00000100:00000001:0.0:1788573740.868233:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573740.868234:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868235:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.868237:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6a300 x1875440810161152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868241:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573740.868242:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573740.868243:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573740.868244:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573740.868245:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573740.868246:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573740.868247:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573740.868248:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573740.868249:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573740.868252:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573740.868254:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6a300. 00000100:00000001:0.0:1788573740.868255:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573740.868256:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868257:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.868260:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6b480 x1875440810161280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868264:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573740.868265:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573740.868266:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573740.868267:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573740.868267:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573740.868268:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573740.868269:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573740.868270:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573740.868271:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573740.868272:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573740.868273:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6b480. 00000100:00000001:0.0:1788573740.868274:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573740.868275:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868276:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573740.868277:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68380 x1875440810161408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573740.868281:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573740.868282:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573740.868283:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573740.868284:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573740.868284:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573740.868285:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573740.868286:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573740.868287:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573740.868288:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573740.868289:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573740.868289:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68380. 00000100:00000001:0.0:1788573740.868291:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573740.868292:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868298:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.868300:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573740.868301:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.868302:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573740.868303:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573743.297857:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573743.297862:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573743.297865:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573744.705804:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308883679, at 4308883712 00000100:00000010:0.0:1788573744.705821:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573745.986075:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573745.986084:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573745.986093:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573745.986112:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.986129:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.986134:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.986145:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573745.986152:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573745.986159:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.986173:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810161536:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573745.986201:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573745.986212:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573745.986218:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.986226:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573745.986236:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573745.986248:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.986273:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573745.986285:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573745.986302:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573745.986313:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161536, portal 25 00000100:00000001:0.0:1788573745.986318:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573745.986322:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573745.986339:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.986379:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573745.986397:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573745.986410:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573745.986425:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810161536, offset 0 00000400:00000200:0.0:1788573745.986443:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573745.986459:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.986489:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573745.986506:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573745.986522:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573745.986537:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573745.986552:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573745.986569:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541353:541353:256:4294967295] 192.168.202.148@tcp LPNI seq info [541318:541318:8:4294967295] 00000400:00000200:0.0:1788573745.986590:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.986620:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573745.986683:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.986690:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573745.986705:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573745.986710:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573745.986749:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.986756:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573745.986768:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573745.986785:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573745.986801:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573745.986809:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573745.986823:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573745.986836:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.986880:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.986890:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.986962:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.986967:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573745.986978:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.986995:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573745.987004:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308888799 00000100:00000010:0.0:1788573745.987022:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573745.987032:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573745.987044:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308888799 00000100:00000010:0.0:1788573745.987054:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626600. 00000100:00000001:0.0:1788573745.987063:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573745.987074:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573745.987081:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.987092:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573745.987103:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573745/real 1788573745] req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573745.987143:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573745.987149:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573745.987157:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573745.987165:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573745.987168:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573745.987174:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573745.987190:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573745.987243:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987289:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573745.987294:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987309:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.987317:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573745.987328:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987363:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573745.987372:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987410:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573745.987427:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987433:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.987440:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987447:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573745.987452:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573745.987474:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573745.987485:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573745.987489:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573745.987502:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573745.987514:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810161536:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573745.987518:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987520:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.987522:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810161536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987528:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573745.987532:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573745.987536:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573745.987540:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573745.987543:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573745.987544:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573745.987547:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573745.987550:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788573745.987554:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573745.987562:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573745.987564:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573745.987568:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573745.987570:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987573:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987582:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573745.987584:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.987586:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573745.987588:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573745.987590:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573745.987593:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987599:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.987600:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.987602:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573745.987604:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573745.987606:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.987608:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573745.987611:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573745.987613:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573745.987614:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.987615:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573745.987618:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573745.987619:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.987623:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573745.987625:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573745.987628:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573745.987631:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161664, portal 10 00000100:00000001:0.0:1788573745.987632:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573745.987633:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286335232 : -109714423216384 : ffff9c371ec70700) 00000100:00000040:0.0:1788573745.987658:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987665:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573745.987668:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573745.987674:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573745.987676:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810161664, offset 0 00000400:00000200:0.0:1788573745.987680:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573745.987684:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.987687:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573745.987690:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573745.987692:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573745.987694:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573745.987696:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573745.987698:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541354:541354:256:4294967295] 192.168.202.148@tcp LPNI seq info [541319:541319:8:4294967295] 00000400:00000200:0.0:1788573745.987701:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.987706:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573745.987709:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.987712:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573745.987715:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573745.987717:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573745.987726:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.987727:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573745.987730:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573745.987732:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573745.987734:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573745.987736:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573745.987738:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573745.987740:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987746:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.987748:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987753:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.987754:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573745.987760:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573745.987764:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573745.987766:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573745.987768:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885018 00000100:00000001:0.0:1788573745.987770:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.987772:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885018 00000100:00000001:0.0:1788573745.987774:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.987776:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573745.987777:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.987778:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573745.987781:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573745/real 1788573745] req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573745.987786:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573745.987787:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573745.987789:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573745.987790:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573745.987792:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573745.987793:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573745.987795:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573745.987801:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987805:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573745.987806:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987807:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.987808:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573745.987810:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987815:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573745.987819:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987832:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573745.987835:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987837:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.987839:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.987841:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573745.987843:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573745.987846:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573745.987850:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573745.987853:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573745.987858:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573745.987867:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161664:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573745.987872:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573745.987875:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987880:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.987881:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.987883:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573745.987884:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573745.987885:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.987887:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161792:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573745.987890:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573745.987892:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573745.987893:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.987894:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573745.987897:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573745.987898:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.987906:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573745.987908:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573745.987911:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573745.987914:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161792, portal 10 00000100:00000001:0.0:1788573745.987916:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573745.987918:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345088 : -109714423206528 : ffff9c371ec72d80) 00000100:00000040:0.0:1788573745.987925:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.987932:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573745.987934:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573745.987937:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573745.987940:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810161792, offset 0 00000400:00000200:0.0:1788573745.987944:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573745.987949:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.987954:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573745.987957:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573745.987960:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573745.987963:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573745.987965:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573745.987969:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541355:541355:256:4294967295] 192.168.202.148@tcp LPNI seq info [541320:541320:8:4294967295] 00000400:00000200:0.0:1788573745.987973:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.987978:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573745.987982:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.987984:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573745.987987:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573745.987990:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573745.987998:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.987999:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573745.988001:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573745.988004:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573745.988007:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573745.988012:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573745.988015:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573745.988018:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988024:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988026:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988031:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988032:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573745.988035:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573745.988038:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573745.988040:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573745.988042:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885022 00000100:00000001:0.0:1788573745.988044:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988045:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885022 00000100:00000001:0.0:1788573745.988047:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988048:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573745.988049:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988051:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573745.988053:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573745/real 1788573745] req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573745.988058:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573745.988059:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573745.988061:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573745.988063:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573745.988064:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573745.988065:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573745.988067:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573745.988073:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988082:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573745.988083:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988084:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988085:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573745.988088:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988093:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573745.988096:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988100:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573745.988101:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988102:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988103:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988104:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573745.988105:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573745.988107:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573745.988109:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573745.988110:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573745.988113:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573745.988117:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161792:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573745.988121:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573745.988123:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988128:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.988129:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.988134:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573745.988135:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573745.988137:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.988139:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573745.988142:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573745.988143:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573745.988145:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.988146:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988148:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573745.988149:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.988152:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573745.988154:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573745.988157:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573745.988159:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810161920, portal 4 00000100:00000001:0.0:1788573745.988160:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573745.988161:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286338816 : -109714423212800 : ffff9c371ec71500) 00000100:00000040:0.0:1788573745.988166:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988171:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573745.988173:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573745.988174:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573745.988176:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810161920, offset 0 00000400:00000200:0.0:1788573745.988178:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573745.988181:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.988183:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573745.988185:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573745.988186:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573745.988188:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573745.988190:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573745.988194:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541356:541356:256:4294967295] 192.168.202.148@tcp LPNI seq info [541321:541321:8:4294967295] 00000400:00000200:0.0:1788573745.988198:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.988205:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573745.988232:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.988236:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573745.988241:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573745.988244:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573745.988255:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.988256:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573745.988259:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573745.988261:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573745.988264:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573745.988265:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573745.988268:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573745.988270:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988276:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988278:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988283:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988284:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573745.988286:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573745.988289:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573745.988291:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573745.988293:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885154 00000100:00000001:0.0:1788573745.988295:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988296:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885154 00000100:00000001:0.0:1788573745.988298:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988299:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573745.988301:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988302:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573745.988304:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573745/real 1788573745] req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573745.988315:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573745.988316:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573745.988317:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573745.988319:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573745.988320:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573745.988322:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573745.988324:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573745.988330:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988334:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573745.988336:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988337:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988338:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573745.988340:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988344:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573745.988346:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988350:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573745.988352:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988353:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988353:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988355:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573745.988357:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573745.988359:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573745.988361:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573745.988363:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573745.988369:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573745.988375:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810161920:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573745.988377:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573745.988379:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988383:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.988384:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.988386:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573745.988387:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573745.988388:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.988390:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162048:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573745.988392:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573745.988393:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573745.988395:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573745.988396:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988398:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573745.988400:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573745.988402:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573745.988404:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573745.988406:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573745.988409:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162048, portal 4 00000100:00000001:0.0:1788573745.988410:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573745.988411:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286336128 : -109714423215488 : ffff9c371ec70a80) 00000100:00000040:0.0:1788573745.988415:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988420:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573745.988422:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573745.988423:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573745.988425:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810162048, offset 0 00000400:00000200:0.0:1788573745.988431:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573745.988434:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.988436:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573745.988438:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573745.988440:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573745.988442:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573745.988444:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573745.988446:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541357:541357:256:4294967295] 192.168.202.148@tcp LPNI seq info [541322:541322:8:4294967295] 00000400:00000200:0.0:1788573745.988449:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573745.988453:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573745.988456:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.988458:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573745.988461:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573745.988476:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 9 secs later) 00000800:00000100:0.0:1788573745.988485:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573745.988486:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573745.988489:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573745.988491:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573745.988493:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573745.988495:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573745.988496:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573745.988499:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988505:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988507:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988511:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988512:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573745.988514:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573745.988517:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573745.988519:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573745.988527:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885163 00000100:00000001:0.0:1788573745.988529:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988531:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885163 00000100:00000001:0.0:1788573745.988534:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308888799 00000100:00000001:0.0:1788573745.988536:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573745.988538:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988540:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573745.988545:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573745/real 1788573745] req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573745.988553:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573745.988555:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573745.988557:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573745.988560:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573745.988562:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573745.988564:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573745.988568:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573745.988578:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988586:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573745.988587:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988588:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988589:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573745.988592:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988597:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573745.988600:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988604:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573745.988610:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988611:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573745.988612:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988614:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573745.988615:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573745.988617:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573745.988619:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573745.988620:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573745.988623:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573745.988629:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162048:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573745.988631:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988632:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988649:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70700 x1875440810161664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988654:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573745.988656:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988657:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573745.988660:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573745.988662:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573745.988663:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573745.988665:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573745.988667:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573745.988670:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573745.988671:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573745.988672:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70700. 00000100:00000001:0.0:1788573745.988674:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573745.988675:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988676:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988679:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72d80 x1875440810161792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988691:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573745.988692:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988693:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573745.988695:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573745.988696:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573745.988697:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573745.988698:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573745.988699:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573745.988700:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573745.988701:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573745.988702:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72d80. 00000100:00000001:0.0:1788573745.988703:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573745.988704:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988705:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988707:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71500 x1875440810161920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988711:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573745.988712:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988713:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573745.988714:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573745.988714:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573745.988715:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573745.988716:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573745.988717:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573745.988718:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573745.988719:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573745.988719:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71500. 00000100:00000001:0.0:1788573745.988721:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573745.988722:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988723:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573745.988725:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70a80 x1875440810162048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573745.988730:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573745.988730:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573745.988734:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e400. 02000000:00000001:0.0:1788573745.988735:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573745.988736:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573745.988737:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573745.988739:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573745.988740:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788573745.988741:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573745.988742:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573745.988742:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70a80. 00000100:00000001:0.0:1788573745.988744:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573745.988745:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988746:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.988748:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573745.988749:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.988750:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573745.988751:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.946575:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573746.946582:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573746.946586:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573746.946594:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.946612:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.946616:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.946623:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573746.946628:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573746.946661:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.946668:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810162304:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573746.946676:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573746.946683:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573746.946688:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.946694:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573746.946703:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.946708:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.946720:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573746.946728:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573746.946737:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573746.946751:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162304, portal 25 00000100:00000001:0.0:1788573746.946756:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573746.946758:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286336128 : -109714423215488 : ffff9c371ec70a80) 00000100:00000040:0.0:1788573746.946772:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.946779:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573746.946784:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573746.946789:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573746.946794:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810162304, offset 0 00000400:00000200:0.0:1788573746.946803:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573746.946812:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.946819:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573746.946827:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573746.946833:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573746.946838:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573746.946844:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573746.946849:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541358:541358:256:4294967295] 192.168.202.148@tcp LPNI seq info [541323:541323:8:4294967295] 00000400:00000200:0.0:1788573746.946859:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.946871:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573746.946879:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.946885:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573746.946902:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573746.946907:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573746.946936:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.946939:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573746.946944:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573746.946953:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573746.946961:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573746.946965:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573746.946976:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573746.946983:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947002:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.947006:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947016:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.947018:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573746.947024:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.947031:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573746.947038:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308889759 00000100:00000001:0.0:1788573746.947046:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4308888799 00000100:00000001:0.0:1788573746.947058:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308889759 00000100:00000001:0.0:1788573746.947065:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4308888799 00000100:00000001:0.0:1788573746.947072:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573746.947075:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.947082:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573746.947089:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573746/real 1788573746] req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573746.947102:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573746.947104:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573746.947108:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573746.947112:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573746.947113:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573746.947118:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573746.947122:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573746.947134:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947153:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573746.947156:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947158:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.947166:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573746.947173:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947192:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573746.947201:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947211:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573746.947215:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947218:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.947220:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947223:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573746.947225:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573746.947227:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573746.947231:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573746.947242:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573746.947254:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573746.947275:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810162304:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573746.947290:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947295:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.947303:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70a80 x1875440810162304/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947325:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573746.947330:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573746.947334:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.947339:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573746.947342:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573746.947345:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573746.947348:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573746.947357:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788573746.947361:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573746.947364:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573746.947368:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70a80. 00000100:00000001:0.0:1788573746.947372:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573746.947375:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947379:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947396:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573746.947399:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.947404:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573746.947409:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573746.947413:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573746.947422:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947459:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.947462:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.947466:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573746.947470:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573746.947475:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.947480:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162432:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573746.947490:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573746.947494:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573746.947497:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.947501:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573746.947508:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.947513:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.947523:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573746.947529:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573746.947537:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573746.947543:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162432, portal 10 00000100:00000001:0.0:1788573746.947546:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573746.947550:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286338816 : -109714423212800 : ffff9c371ec71500) 00000100:00000040:0.0:1788573746.947560:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947587:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573746.947591:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573746.947594:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573746.947599:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810162432, offset 0 00000400:00000200:0.0:1788573746.947610:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573746.947621:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.947631:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573746.947661:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573746.947667:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573746.947674:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573746.947679:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573746.947685:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541359:541359:256:4294967295] 192.168.202.148@tcp LPNI seq info [541324:541324:8:4294967295] 00000400:00000200:0.0:1788573746.947694:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.947704:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573746.947712:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.947717:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573746.947726:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573746.947729:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573746.947745:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.947748:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573746.947754:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573746.947765:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573746.947772:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573746.947778:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573746.947784:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573746.947793:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947806:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.947811:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947827:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.947828:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573746.947832:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573746.947835:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573746.947838:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573746.947840:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885978 00000100:00000001:0.0:1788573746.947842:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.947843:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885978 00000100:00000001:0.0:1788573746.947845:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.947846:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573746.947848:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.947849:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573746.947852:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573746/real 1788573746] req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573746.947857:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573746.947858:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573746.947859:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573746.947861:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573746.947862:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573746.947863:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573746.947865:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573746.947871:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947875:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573746.947876:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947877:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.947878:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573746.947880:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947888:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573746.947891:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947895:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573746.947897:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947898:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.947899:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.947900:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573746.947901:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573746.947903:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573746.947905:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573746.947907:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573746.947909:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573746.947914:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162432:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573746.947916:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573746.947918:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947922:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.947923:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.947925:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573746.947926:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573746.947927:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.947929:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162560:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573746.947931:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573746.947933:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573746.947937:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.947938:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573746.947940:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573746.947941:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.947944:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573746.947946:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573746.947948:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573746.947950:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162560, portal 10 00000100:00000001:0.0:1788573746.947951:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573746.947952:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286345088 : -109714423206528 : ffff9c371ec72d80) 00000100:00000040:0.0:1788573746.947957:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.947961:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573746.947963:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573746.947964:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573746.947966:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810162560, offset 0 00000400:00000200:0.0:1788573746.947968:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573746.947971:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.947973:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573746.947975:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573746.947977:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573746.947978:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573746.947980:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573746.947982:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541360:541360:256:4294967295] 192.168.202.148@tcp LPNI seq info [541325:541325:8:4294967295] 00000400:00000200:0.0:1788573746.947985:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.947989:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573746.947992:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.947994:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573746.947996:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573746.947998:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573746.948006:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.948010:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573746.948012:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573746.948015:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573746.948017:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573746.948018:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573746.948020:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573746.948022:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948028:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948030:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948034:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948035:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573746.948037:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573746.948039:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573746.948041:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573746.948042:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308885982 00000100:00000001:0.0:1788573746.948044:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948045:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308885982 00000100:00000001:0.0:1788573746.948047:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948048:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573746.948049:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948050:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573746.948053:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573746/real 1788573746] req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573746.948060:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573746.948061:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573746.948063:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573746.948065:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573746.948067:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573746.948074:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573746.948078:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573746.948088:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948095:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573746.948096:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948099:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948100:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573746.948104:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948111:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573746.948114:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948119:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573746.948120:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948121:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948122:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948123:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573746.948124:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573746.948126:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573746.948128:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573746.948129:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573746.948132:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573746.948137:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162560:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573746.948140:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948146:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948148:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71500 x1875440810162432/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948153:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573746.948154:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948156:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.948157:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573746.948158:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573746.948159:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573746.948161:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573746.948162:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788573746.948163:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573746.948164:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573746.948165:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71500. 00000100:00000001:0.0:1788573746.948168:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573746.948169:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948171:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948174:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72d80 x1875440810162560/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948180:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573746.948182:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948183:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573746.948186:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573746.948187:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573746.948188:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573746.948190:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573746.948192:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788573746.948194:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573746.948195:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573746.948197:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72d80. 00000100:00000001:0.0:1788573746.948200:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573746.948201:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948203:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948207:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573746.948209:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948211:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573746.948217:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573746.948218:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573746.948222:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948226:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.948227:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.948229:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573746.948231:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573746.948232:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.948234:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162688:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573746.948236:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573746.948237:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573746.948238:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.948240:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948241:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573746.948243:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.948246:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573746.948247:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573746.948250:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573746.948252:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162688, portal 4 00000100:00000001:0.0:1788573746.948254:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573746.948255:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286335232 : -109714423216384 : ffff9c371ec70700) 00000100:00000040:0.0:1788573746.948259:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948263:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573746.948264:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573746.948266:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573746.948267:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810162688, offset 0 00000400:00000200:0.0:1788573746.948270:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573746.948272:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.948275:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573746.948277:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573746.948279:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573746.948284:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573746.948286:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573746.948288:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541361:541361:256:4294967295] 192.168.202.148@tcp LPNI seq info [541326:541326:8:4294967295] 00000400:00000200:0.0:1788573746.948291:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.948296:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573746.948299:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.948301:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573746.948304:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573746.948306:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573746.948313:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.948314:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573746.948316:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573746.948319:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573746.948321:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573746.948322:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573746.948323:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573746.948326:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948331:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948333:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948338:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948339:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573746.948341:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573746.948343:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573746.948345:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573746.948346:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308886114 00000100:00000001:0.0:1788573746.948348:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948349:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308886114 00000100:00000001:0.0:1788573746.948355:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948356:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573746.948357:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948359:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573746.948362:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573746/real 1788573746] req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573746.948367:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573746.948368:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573746.948369:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573746.948370:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573746.948372:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573746.948373:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573746.948375:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573746.948381:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948388:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573746.948389:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948391:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948392:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573746.948396:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948403:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573746.948407:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948415:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573746.948418:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948420:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948421:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948428:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573746.948430:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573746.948433:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573746.948436:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573746.948438:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573746.948441:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573746.948461:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162688:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573746.948464:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573746.948466:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948471:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.948472:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.948474:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573746.948475:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573746.948476:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.948478:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162816:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573746.948480:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573746.948481:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573746.948482:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573746.948484:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948486:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.948487:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573746.948490:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573746.948493:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573746.948496:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573746.948499:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162816, portal 4 00000100:00000001:0.0:1788573746.948501:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573746.948503:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788573746.948509:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948521:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573746.948525:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573746.948527:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573746.948530:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810162816, offset 0 00000400:00000200:0.0:1788573746.948534:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573746.948539:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.948542:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573746.948544:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573746.948546:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573746.948548:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573746.948549:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573746.948552:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541362:541362:256:4294967295] 192.168.202.148@tcp LPNI seq info [541327:541327:8:4294967295] 00000400:00000200:0.0:1788573746.948555:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573746.948559:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573746.948562:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.948565:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573746.948567:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573746.948569:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 8 secs later) 00000800:00000100:0.0:1788573746.948576:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573746.948578:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573746.948580:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573746.948582:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573746.948584:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573746.948585:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573746.948587:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573746.948590:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948598:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948601:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948620:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948621:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573746.948626:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573746.948629:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573746.948633:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573746.948653:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308886123 00000100:00000001:0.0:1788573746.948656:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948657:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308886123 00000100:00000001:0.0:1788573746.948659:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308889759 00000100:00000001:0.0:1788573746.948661:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573746.948662:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948665:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573746.948669:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573746/real 1788573746] req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573746.948675:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573746.948677:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573746.948679:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573746.948681:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573746.948682:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573746.948683:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573746.948685:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573746.948691:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948695:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573746.948696:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948697:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948697:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573746.948700:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948710:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573746.948713:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948717:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573746.948719:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948720:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573746.948721:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948722:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573746.948723:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573746.948725:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573746.948726:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573746.948728:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573746.948730:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573746.948735:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810162816:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573746.948737:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948738:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948740:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70700 x1875440810162688/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948745:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573746.948747:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948749:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573746.948752:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573746.948753:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573746.948755:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573746.948757:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573746.948759:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788573746.948761:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573746.948767:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573746.948770:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70700. 00000100:00000001:0.0:1788573746.948773:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573746.948774:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948776:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573746.948780:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810162816/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573746.948787:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573746.948788:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573746.948790:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20ec00. 02000000:00000001:0.0:1788573746.948793:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573746.948794:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573746.948795:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573746.948797:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573746.948798:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788573746.948800:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573746.948800:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573746.948801:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788573746.948803:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573746.948804:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948805:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.948808:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573746.948809:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.948810:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573746.948811:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573748.417831:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573748.417834:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573748.417837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573750.785725:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308889759, at 4308889792 00000100:00000010:0.0:1788573750.785745:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573752.129974:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573752.129980:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573752.129992:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573752.130002:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130020:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.130031:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.130038:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573752.130042:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573752.130046:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.130054:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810162944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573752.130062:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573752.130067:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573752.130069:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.130072:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573752.130078:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573752.130084:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.130092:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573752.130102:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573752.130114:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573752.130126:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810162944, portal 25 00000100:00000001:0.0:1788573752.130131:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573752.130136:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287364480 : -109714422187136 : ffff9c371ed6bb80) 00000100:00000040:0.0:1788573752.130152:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130170:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573752.130178:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573752.130183:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573752.130188:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810162944, offset 0 00000400:00000200:0.0:1788573752.130198:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573752.130209:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.130227:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573752.130236:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573752.130246:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573752.130254:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573752.130260:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573752.130267:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541363:541363:256:4294967295] 192.168.202.148@tcp LPNI seq info [541328:541328:8:4294967295] 00000400:00000200:0.0:1788573752.130275:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.130288:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573752.130309:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.130319:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573752.130329:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573752.130336:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573752.130376:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.130381:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573752.130389:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573752.130398:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573752.130405:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573752.130410:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573752.130425:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573752.130433:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130457:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.130465:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130480:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.130483:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573752.130488:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.130494:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573752.130499:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308894942 00000100:00000010:0.0:1788573752.130505:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573752.130514:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573752.130524:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308894942 00000100:00000010:0.0:1788573752.130531:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116268a0. 00000100:00000001:0.0:1788573752.130534:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573752.130538:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573752.130544:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.130549:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573752.130556:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573752/real 1788573752] req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573752.130596:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573752.130599:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573752.130603:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573752.130609:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573752.130613:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573752.130619:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573752.130626:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573752.130688:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130709:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573752.130713:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130718:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.130721:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573752.130730:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130741:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573752.130749:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130764:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573752.130769:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130774:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.130778:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130781:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573752.130785:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573752.130789:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573752.130798:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573752.130808:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573752.130816:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573752.130845:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810162944:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573752.130857:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130863:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.130870:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6bb80 x1875440810162944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130878:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573752.130882:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573752.130885:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573752.130888:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573752.130889:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573752.130891:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573752.130893:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573752.130895:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37800. 02000000:00000001:0.0:1788573752.130898:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573752.130899:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573752.130901:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6bb80. 00000100:00000001:0.0:1788573752.130904:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573752.130905:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130907:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.130913:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573752.130914:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.130916:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573752.130917:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573752.130962:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573752.130968:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.130976:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.130978:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.130981:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573752.130983:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573752.130986:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.130991:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573752.131003:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573752.131006:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573752.131007:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131009:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131012:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573752.131014:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.131019:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573752.131022:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573752.131026:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573752.131030:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163072, portal 10 00000100:00000001:0.0:1788573752.131031:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573752.131032:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287357312 : -109714422194304 : ffff9c371ed69f80) 00000100:00000040:0.0:1788573752.131038:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131042:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573752.131044:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573752.131045:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573752.131047:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810163072, offset 0 00000400:00000200:0.0:1788573752.131050:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131053:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131055:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573752.131058:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573752.131060:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573752.131062:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573752.131063:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131065:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541364:541364:256:4294967295] 192.168.202.148@tcp LPNI seq info [541329:541329:8:4294967295] 00000400:00000200:0.0:1788573752.131068:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131073:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573752.131076:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131078:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573752.131081:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573752.131087:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573752.131097:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131099:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573752.131102:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573752.131106:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573752.131109:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573752.131111:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573752.131114:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573752.131118:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131127:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131130:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131138:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131139:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573752.131143:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573752.131147:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573752.131151:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573752.131153:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308891161 00000100:00000001:0.0:1788573752.131155:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131156:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308891161 00000100:00000001:0.0:1788573752.131158:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131160:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573752.131161:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131163:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573752.131165:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573752/real 1788573752] req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573752.131171:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573752.131172:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573752.131177:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573752.131179:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573752.131180:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573752.131182:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573752.131185:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573752.131190:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131195:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573752.131196:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131197:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131198:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573752.131200:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131204:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573752.131206:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131211:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573752.131213:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131214:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131215:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131216:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573752.131217:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573752.131219:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573752.131221:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573752.131222:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573752.131224:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573752.131229:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163072:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573752.131235:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573752.131237:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131242:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.131243:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131244:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573752.131245:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573752.131247:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.131249:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573752.131251:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573752.131252:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573752.131253:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131255:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131257:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573752.131258:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.131260:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573752.131262:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573752.131264:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573752.131266:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163200, portal 10 00000100:00000001:0.0:1788573752.131268:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573752.131269:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287355520 : -109714422196096 : ffff9c371ed69880) 00000100:00000040:0.0:1788573752.131273:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131277:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573752.131279:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573752.131280:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573752.131282:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810163200, offset 0 00000400:00000200:0.0:1788573752.131284:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131287:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131290:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573752.131292:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573752.131298:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573752.131301:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573752.131303:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131306:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541365:541365:256:4294967295] 192.168.202.148@tcp LPNI seq info [541330:541330:8:4294967295] 00000400:00000200:0.0:1788573752.131310:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131317:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573752.131322:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131326:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573752.131330:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573752.131333:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573752.131357:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131359:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573752.131362:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573752.131364:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573752.131367:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573752.131368:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573752.131370:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573752.131373:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131379:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131381:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131385:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131386:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573752.131389:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573752.131391:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573752.131393:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573752.131395:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308891165 00000100:00000001:0.0:1788573752.131396:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131398:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308891165 00000100:00000001:0.0:1788573752.131404:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131406:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573752.131407:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131409:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573752.131411:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573752/real 1788573752] req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573752.131417:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573752.131418:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573752.131419:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573752.131421:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573752.131422:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573752.131423:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573752.131425:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573752.131430:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131434:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573752.131435:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131436:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131437:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573752.131439:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131443:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573752.131446:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131450:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573752.131452:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131453:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131453:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131457:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573752.131459:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573752.131461:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573752.131463:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573752.131465:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573752.131467:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573752.131472:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163200:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573752.131475:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573752.131477:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131481:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.131482:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131483:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573752.131484:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573752.131486:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.131487:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573752.131489:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573752.131490:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573752.131491:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131492:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131494:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573752.131496:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.131498:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573752.131500:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573752.131502:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573752.131504:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163328, portal 4 00000100:00000001:0.0:1788573752.131505:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573752.131506:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287354624 : -109714422196992 : ffff9c371ed69500) 00000100:00000040:0.0:1788573752.131510:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131519:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573752.131521:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573752.131522:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573752.131524:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810163328, offset 0 00000400:00000200:0.0:1788573752.131527:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131529:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131531:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573752.131533:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573752.131535:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573752.131537:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573752.131538:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131541:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541366:541366:256:4294967295] 192.168.202.148@tcp LPNI seq info [541331:541331:8:4294967295] 00000400:00000200:0.0:1788573752.131543:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131548:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573752.131550:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131553:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573752.131555:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573752.131557:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573752.131564:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131566:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573752.131568:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573752.131570:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573752.131571:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573752.131573:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573752.131574:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573752.131576:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131582:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131584:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131595:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131596:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573752.131599:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573752.131601:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573752.131603:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573752.131604:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308891297 00000100:00000001:0.0:1788573752.131606:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131608:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308891297 00000100:00000001:0.0:1788573752.131609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131611:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573752.131612:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131613:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573752.131615:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573752/real 1788573752] req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573752.131620:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573752.131621:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573752.131622:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573752.131623:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573752.131624:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573752.131626:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573752.131628:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573752.131633:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131652:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573752.131653:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131654:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131655:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573752.131658:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131667:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573752.131669:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131674:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573752.131676:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131677:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131677:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131679:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573752.131680:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573752.131682:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573752.131684:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573752.131685:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573752.131687:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573752.131692:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163328:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573752.131694:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573752.131696:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131700:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.131701:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131703:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573752.131704:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573752.131705:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.131707:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573752.131709:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573752.131710:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573752.131714:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573752.131715:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131717:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f400. 02000000:00000001:0.0:1788573752.131718:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573752.131721:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573752.131723:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573752.131725:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573752.131727:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163456, portal 4 00000100:00000001:0.0:1788573752.131728:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573752.131729:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287352832 : -109714422198784 : ffff9c371ed68e00) 00000100:00000040:0.0:1788573752.131734:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131738:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573752.131740:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573752.131741:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573752.131742:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810163456, offset 0 00000400:00000200:0.0:1788573752.131745:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131747:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131749:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573752.131751:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573752.131753:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573752.131754:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573752.131756:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573752.131758:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541367:541367:256:4294967295] 192.168.202.148@tcp LPNI seq info [541332:541332:8:4294967295] 00000400:00000200:0.0:1788573752.131760:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573752.131764:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573752.131767:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131769:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573752.131772:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573752.131773:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 3 secs later) 00000800:00000100:0.0:1788573752.131780:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573752.131820:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0800. 00000400:00000200:0.0:1788573752.131823:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573752.131826:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573752.131828:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31880 00000400:00000010:0.0:1788573752.131829:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31880. 00000100:00000001:0.0:1788573752.131831:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573752.131834:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131839:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131841:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131845:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131846:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573752.131848:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573752.131850:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573752.131852:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573752.131853:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308891307 00000100:00000001:0.0:1788573752.131855:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131857:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308891307 00000100:00000001:0.0:1788573752.131858:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308894942 00000100:00000001:0.0:1788573752.131860:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573752.131861:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.131862:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573752.131864:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573752/real 1788573752] req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573752.131869:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573752.131870:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573752.131871:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573752.131872:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573752.131873:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573752.131878:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573752.131881:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573752.131886:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131890:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573752.131891:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131892:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131893:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573752.131895:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131899:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573752.131902:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131906:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573752.131907:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131908:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573752.131909:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131910:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573752.131911:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573752.131913:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573752.131915:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573752.131916:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573752.131918:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573752.131923:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163456:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573752.131925:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131926:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131931:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69f80 x1875440810163072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131936:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573752.131937:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131938:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573752.131940:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573752.131941:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573752.131942:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573752.131944:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573752.131945:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35c00. 02000000:00000001:0.0:1788573752.131947:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573752.131948:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573752.131949:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69f80. 00000100:00000001:0.0:1788573752.131951:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573752.131952:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131953:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131955:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69880 x1875440810163200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131959:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573752.131960:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131961:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573752.131962:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573752.131963:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573752.131963:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573752.131965:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573752.131965:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37400. 02000000:00000001:0.0:1788573752.131967:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573752.131967:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573752.131968:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69880. 00000100:00000001:0.0:1788573752.131969:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573752.131970:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131971:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131973:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69500 x1875440810163328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131977:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573752.131981:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573752.131982:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788573752.131984:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573752.131984:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573752.131985:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573752.131987:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573752.131988:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34000. 02000000:00000001:0.0:1788573752.131989:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573752.131990:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573752.131990:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69500. 00000100:00000001:0.0:1788573752.131992:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573752.131992:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.131993:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573752.131996:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68e00 x1875440810163456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573752.131999:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573752.132000:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573752.132001:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f400. 02000000:00000001:0.0:1788573752.132002:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573752.132003:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573752.132003:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573752.132004:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573752.132005:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35000. 02000000:00000001:0.0:1788573752.132006:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573752.132007:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573752.132008:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68e00. 00000100:00000001:0.0:1788573752.132009:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573752.132010:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.132011:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.132013:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573752.132014:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.132016:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573752.132017:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573753.537819:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573753.537823:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573753.537825:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573755.969740:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308894942, at 4308894976 00000100:00000010:0.0:1788573755.969769:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573757.250080:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573757.250087:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.250091:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573757.250099:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.250113:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.250116:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.250121:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573757.250125:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573757.250128:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.250135:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810163584:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573757.250141:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573757.250147:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573757.250152:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.250157:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573757.250163:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573757.250168:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.250183:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573757.250188:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8d10. 00000400:00000010:0.0:1788573757.250195:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31908. 00000100:00000200:0.0:1788573757.250203:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163584, portal 25 00000100:00000001:0.0:1788573757.250209:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573757.250211:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788573757.250219:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.250228:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573757.250233:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573757.250238:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31880. 00000100:00000200:0.0:1788573757.250256:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810163584, offset 0 00000400:00000200:0.0:1788573757.250263:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573757.250272:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.250280:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573757.250292:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573757.250297:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573757.250303:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573757.250308:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573757.250315:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541368:541368:256:4294967295] 192.168.202.148@tcp LPNI seq info [541333:541333:8:4294967295] 00000400:00000200:0.0:1788573757.250326:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.250341:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573757.250349:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573757.250356:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0800. 00000800:00000200:0.0:1788573757.250367:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573757.250393:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.250403:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573757.250409:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308900062 00000100:00000010:0.0:1788573757.250419:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573757.250428:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573757.250438:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308900062 00000100:00000010:0.0:1788573757.250446:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626d20. 00000100:00000001:0.0:1788573757.250453:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573757.250460:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573757.250469:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.250476:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.250481:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.250491:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573757.250496:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573757.250501:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573757.250505:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.250509:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573757.250517:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.250532:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.250535:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.250547:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573757.250550:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573757.250554:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.250561:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163712:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573757.250568:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573757.250572:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573757.250575:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.250580:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573757.250587:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573757.250592:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.250601:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573757.250604:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573757.250610:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573757.250615:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163712, portal 10 00000100:00000001:0.0:1788573757.250620:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573757.250622:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788573757.250631:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.250678:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573757.250683:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573757.250688:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed316e8. 00000100:00000200:0.0:1788573757.250695:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810163712, offset 0 00000400:00000200:0.0:1788573757.250707:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573757.250716:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.250736:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573757.250744:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573757.250752:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573757.250759:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573757.250767:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573757.250775:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541369:541369:255:4294967295] 192.168.202.148@tcp LPNI seq info [541334:541334:7:4294967295] 00000400:00000200:0.0:1788573757.250802:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.250819:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573757.250839:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573757.250847:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0100. 00000800:00000200:0.0:1788573757.250857:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573757.250871:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573757.250880:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573757.250890:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573757.250896:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308896281 00000100:00000001:0.0:1788573757.250904:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.250911:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308896281 00000100:00000001:0.0:1788573757.250918:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.250924:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573757.250930:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.250937:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.250942:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.250948:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573757.250951:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573757.250953:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573757.250956:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.250959:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573757.250965:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.250978:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.250982:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.250988:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573757.250990:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573757.250997:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.251006:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573757.251015:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573757.251019:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573757.251024:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.251028:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573757.251043:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573757.251049:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.251058:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573757.251063:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c87e8. 00000400:00000010:0.0:1788573757.251071:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31aa0. 00000100:00000200:0.0:1788573757.251078:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163840, portal 10 00000100:00000001:0.0:1788573757.251086:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573757.251090:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788573757.251102:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.251118:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573757.251124:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573757.251128:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31088. 00000100:00000200:0.0:1788573757.251135:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810163840, offset 0 00000400:00000200:0.0:1788573757.251147:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251152:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251158:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573757.251167:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573757.251172:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573757.251177:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573757.251183:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251191:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541370:541370:254:4294967295] 192.168.202.148@tcp LPNI seq info [541335:541335:6:4294967295] 00000400:00000200:0.0:1788573757.251206:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251219:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573757.251226:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573757.251232:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0400. 00000800:00000200:0.0:1788573757.251238:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573757.251261:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573757.251264:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573757.251268:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573757.251269:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308896285 00000100:00000001:0.0:1788573757.251280:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251283:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308896285 00000100:00000001:0.0:1788573757.251284:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251286:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573757.251288:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251290:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573757.251295:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.251301:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.251303:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.251305:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573757.251306:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573757.251308:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.251311:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163968:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573757.251314:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573757.251316:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573757.251318:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.251319:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573757.251322:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573757.251324:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.251329:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573757.251331:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8840. 00000400:00000010:0.0:1788573757.251335:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573757.251339:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810163968, portal 4 00000100:00000001:0.0:1788573757.251340:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573757.251342:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788573757.251347:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.251354:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573757.251356:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573757.251358:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31b28. 00000100:00000200:0.0:1788573757.251367:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810163968, offset 0 00000400:00000200:0.0:1788573757.251372:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251376:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251379:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573757.251382:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573757.251384:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573757.251386:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573757.251388:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251391:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541371:541371:253:4294967295] 192.168.202.148@tcp LPNI seq info [541336:541336:5:4294967295] 00000400:00000200:0.0:1788573757.251395:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251401:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573757.251406:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573757.251411:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0e00. 00000800:00000200:0.0:1788573757.251415:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573757.251422:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573757.251425:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573757.251428:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573757.251430:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308896417 00000100:00000001:0.0:1788573757.251433:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251435:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308896417 00000100:00000001:0.0:1788573757.251437:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251439:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573757.251442:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251444:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251446:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.251450:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573757.251452:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573757.251454:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573757.251456:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.251457:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573757.251462:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.251477:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.251479:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.251482:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573757.251483:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573757.251486:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.251489:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573757.251493:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573757.251495:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573757.251496:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573757.251498:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573757.251502:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573757.251504:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573757.251508:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573757.251511:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8b58. 00000400:00000010:0.0:1788573757.251515:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed317f8. 00000100:00000200:0.0:1788573757.251518:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164096, portal 4 00000100:00000001:0.0:1788573757.251520:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573757.251521:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788573757.251528:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573757.251535:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573757.251537:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573757.251539:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31550. 00000100:00000200:0.0:1788573757.251542:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810164096, offset 0 00000400:00000200:0.0:1788573757.251546:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251550:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251553:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573757.251556:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573757.251559:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573757.251562:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573757.251565:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573757.251574:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541372:541372:252:4294967295] 192.168.202.148@tcp LPNI seq info [541337:541337:4:4294967295] 00000400:00000200:0.0:1788573757.251579:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573757.251586:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573757.251591:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573757.251595:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0000. 00000800:00000200:0.0:1788573757.251599:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000100:00000001:0.0:1788573757.251606:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573757.251610:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573757.251614:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573757.251616:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308896426 00000100:00000001:0.0:1788573757.251619:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251621:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308896426 00000100:00000001:0.0:1788573757.251623:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308900062 00000100:00000001:0.0:1788573757.251626:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573757.251628:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251631:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251633:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.251659:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573757.251661:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573757.251663:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573757.251664:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.251666:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573757.251669:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573758.657850:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573758.657853:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573758.657856:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573758.721932:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573760.322058:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573760.322064:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573760.322081:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573760.322094:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573757/real 1788573760] req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573760.322151:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573760.322155:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573760.322159:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8d10. 00000400:00000200:0.0:1788573760.322165:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31908 00000400:00000010:0.0:1788573760.322169:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31908. 00000100:00000001:0.0:1788573760.322174:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573760.322195:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573760.322217:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322237:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573760.322242:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322248:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322252:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573760.322261:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322276:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573760.322287:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322301:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573760.322308:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322312:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322314:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322317:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573760.322321:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573760.322329:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573760.322335:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573760.322358:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573760.322370:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573760.322407:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810163584:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573760.322419:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573760.322424:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573760.322432:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810163584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322456:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573760.322462:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573760.322473:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788573760.322480:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573760.322485:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573760.322488:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573760.322494:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573760.322499:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788573760.322505:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573760.322510:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573760.322514:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788573760.322520:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573760.322524:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322529:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573760.322532:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573760.322533:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573760.322535:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573760.322541:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573757/real 1788573760] req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573760.322552:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573760.322554:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573760.322556:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8b58. 00000400:00000200:0.0:1788573760.322560:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed317f8 00000400:00000010:0.0:1788573760.322562:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed317f8. 00000100:00000001:0.0:1788573760.322565:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573760.322568:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573760.322589:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322596:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573760.322597:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322599:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322601:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573760.322605:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322611:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573760.322615:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322621:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573760.322624:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322625:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322626:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322628:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573760.322630:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573760.322633:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573760.322685:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573760.322688:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573760.322693:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573760.322701:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164096:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573760.322758:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573760.322763:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573757/real 1788573760] req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573760.322788:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573760.322790:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573760.322793:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c87e8. 00000400:00000200:0.0:1788573760.322796:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31aa0 00000400:00000010:0.0:1788573760.322798:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31aa0. 00000100:00000001:0.0:1788573760.322802:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573760.322807:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573760.322818:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322827:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573760.322828:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322830:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322832:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573760.322836:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322843:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573760.322847:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322854:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573760.322857:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322859:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322860:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322862:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573760.322864:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573760.322867:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573760.322871:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573760.322873:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573760.322884:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573760.322893:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163840:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573760.322898:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573760.322902:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573757/real 1788573760] req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573760.322911:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573760.322913:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573760.322915:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8840. 00000400:00000200:0.0:1788573760.322919:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573760.322921:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573760.322923:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573760.322927:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573760.322936:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322942:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573760.322944:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322946:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322948:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573760.322951:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322958:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573760.322963:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.322970:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573760.322972:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322980:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.322981:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.322983:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573760.322986:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573760.322989:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573760.322993:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573760.322995:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573760.322999:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573760.323007:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163968:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573760.323012:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573760.323017:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573757/real 1788573760] req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573760.323025:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573760.323026:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573760.323028:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573760.323031:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573760.323033:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573760.323036:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573760.323038:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573760.323046:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323053:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573760.323054:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323056:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.323057:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573760.323060:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323080:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573760.323085:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323091:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573760.323107:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323109:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573760.323111:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323113:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573760.323115:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573760.323118:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573760.323123:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573760.323126:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573760.323132:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573760.323141:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810163712:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573760.323146:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323148:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573760.323152:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810164096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323160:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573760.323162:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573760.323165:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573760.323169:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573760.323171:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573760.323172:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573760.323174:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573760.323177:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788573760.323200:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573760.323202:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573760.323205:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788573760.323209:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573760.323211:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323213:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573760.323219:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810163840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323227:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573760.323230:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573760.323232:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573760.323236:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573760.323238:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573760.323240:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573760.323242:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573760.323244:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788573760.323247:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573760.323248:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573760.323250:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788573760.323252:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573760.323253:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323259:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573760.323263:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810163968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323270:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573760.323272:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573760.323274:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573760.323277:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573760.323278:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573760.323279:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573760.323281:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573760.323282:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788573760.323284:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573760.323285:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573760.323287:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788573760.323289:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573760.323296:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323298:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573760.323302:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810163712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573760.323308:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573760.323309:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573760.323311:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e800. 02000000:00000001:0.0:1788573760.323314:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573760.323316:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573760.323317:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573760.323319:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573760.323321:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788573760.323324:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573760.323325:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573760.323326:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788573760.323329:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573760.323330:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323331:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.323340:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573760.323342:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573760.323345:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573760.323347:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573761.089721:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308900062, at 4308900096 00000100:00000010:0.0:1788573761.089740:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573762.369992:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573762.369999:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573762.370003:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573762.370013:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370033:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.370039:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.370046:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573762.370051:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573762.370059:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.370067:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810164224:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573762.370088:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573762.370094:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573762.370099:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.370106:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573762.370115:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573762.370127:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.370154:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573762.370161:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573762.370171:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573762.370181:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164224, portal 25 00000100:00000001:0.0:1788573762.370188:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573762.370192:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287351936 : -109714422199680 : ffff9c371ed68a80) 00000100:00000040:0.0:1788573762.370207:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370237:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573762.370246:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573762.370252:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573762.370259:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810164224, offset 0 00000400:00000200:0.0:1788573762.370276:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573762.370289:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.370300:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573762.370309:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573762.370317:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573762.370324:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573762.370331:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573762.370342:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541373:541373:256:4294967295] 192.168.202.148@tcp LPNI seq info [541338:541338:8:4294967295] 00000400:00000200:0.0:1788573762.370353:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.370370:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573762.370383:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.370393:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573762.370405:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573762.370423:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573762.370449:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.370454:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573762.370463:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573762.370475:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573762.370485:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573762.370490:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573762.370497:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573762.370507:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370528:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.370537:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370548:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.370551:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573762.370560:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.370570:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573762.370577:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4308905182 00000100:00000010:0.0:1788573762.370585:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788573762.370593:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573762.370599:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4308905182 00000100:00000010:0.0:1788573762.370603:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626de0. 00000100:00000001:0.0:1788573762.370606:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573762.370610:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573762.370613:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.370615:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573762.370621:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573762/real 1788573762] req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573762.370631:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573762.370632:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573762.370670:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573762.370675:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573762.370677:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573762.370682:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573762.370687:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573762.370699:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370708:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573762.370713:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370729:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.370734:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573762.370744:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370764:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573762.370772:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370788:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573762.370793:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370799:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.370801:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370808:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573762.370813:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573762.370820:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573762.370828:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573762.370836:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573762.370843:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573762.370861:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810164224:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788573762.370885:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370891:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.370900:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68a80 x1875440810164224/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.370920:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573762.370926:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573762.370932:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573762.370939:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573762.370943:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573762.370946:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573762.370951:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573762.370957:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da35800. 02000000:00000001:0.0:1788573762.370963:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573762.370968:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573762.370972:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68a80. 00000100:00000001:0.0:1788573762.370978:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573762.370981:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370986:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.370996:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573762.371000:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371006:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573762.371010:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573762.371012:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573762.371018:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371035:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.371038:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371045:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573762.371049:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573762.371053:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.371064:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164352:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573762.371072:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573762.371077:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573762.371081:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371092:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573762.371098:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573762.371104:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.371113:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573762.371119:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573762.371126:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573762.371134:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164352, portal 10 00000100:00000001:0.0:1788573762.371152:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573762.371153:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287349248 : -109714422202368 : ffff9c371ed68000) 00000100:00000040:0.0:1788573762.371168:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371182:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573762.371189:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573762.371195:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573762.371201:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810164352, offset 0 00000400:00000200:0.0:1788573762.371210:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371217:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371224:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573762.371228:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573762.371232:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573762.371235:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573762.371238:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371241:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541374:541374:256:4294967295] 192.168.202.148@tcp LPNI seq info [541339:541339:8:4294967295] 00000400:00000200:0.0:1788573762.371246:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371253:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573762.371259:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371263:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573762.371268:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573762.371272:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573762.371286:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371288:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573762.371299:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573762.371304:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573762.371309:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573762.371312:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573762.371315:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573762.371320:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371328:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.371331:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371339:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371340:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573762.371344:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573762.371348:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788573762.371352:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573762.371355:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308901401 00000100:00000001:0.0:1788573762.371358:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371360:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308901401 00000100:00000001:0.0:1788573762.371362:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371364:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573762.371366:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371369:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573762.371373:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573762/real 1788573762] req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573762.371381:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573762.371383:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573762.371385:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573762.371388:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573762.371389:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573762.371391:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573762.371401:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573762.371410:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371417:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573762.371419:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371421:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.371422:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573762.371427:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371435:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573762.371440:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371447:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573762.371450:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371452:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.371453:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371455:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573762.371457:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573762.371459:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573762.371463:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573762.371466:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573762.371470:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573762.371478:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164352:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573762.371483:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573762.371486:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371504:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.371506:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371509:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573762.371510:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573762.371513:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.371517:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573762.371520:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573762.371522:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573762.371524:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371525:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573762.371528:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573762.371530:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.371534:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573762.371537:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573762.371541:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573762.371543:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164480, portal 10 00000100:00000001:0.0:1788573762.371545:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573762.371547:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287356416 : -109714422195200 : ffff9c371ed69c00) 00000100:00000040:0.0:1788573762.371553:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371560:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573762.371562:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573762.371564:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573762.371566:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810164480, offset 0 00000400:00000200:0.0:1788573762.371571:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371576:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371579:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573762.371583:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573762.371585:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573762.371587:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573762.371589:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371600:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541375:541375:256:4294967295] 192.168.202.148@tcp LPNI seq info [541340:541340:8:4294967295] 00000400:00000200:0.0:1788573762.371604:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371611:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573762.371615:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371618:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573762.371622:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573762.371624:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573762.371660:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371662:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573762.371666:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573762.371670:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573762.371674:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573762.371676:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573762.371679:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573762.371683:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371691:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.371695:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371701:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371703:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573762.371706:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573762.371709:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788573762.371712:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573762.371714:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308901406 00000100:00000001:0.0:1788573762.371717:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371719:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308901406 00000100:00000001:0.0:1788573762.371721:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371724:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573762.371731:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371733:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573762.371738:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573762/real 1788573762] req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573762.371746:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573762.371747:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573762.371749:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573762.371751:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573762.371752:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573762.371753:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573762.371756:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573762.371762:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371766:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573762.371767:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371768:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.371769:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573762.371771:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371775:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573762.371778:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371782:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573762.371784:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371785:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.371786:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371787:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573762.371789:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573762.371791:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573762.371796:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573762.371798:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573762.371801:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573762.371806:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164480:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788573762.371809:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573762.371811:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371815:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.371816:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371818:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573762.371819:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573762.371820:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.371822:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164608:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573762.371824:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573762.371825:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573762.371826:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.371827:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573762.371830:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573762.371831:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.371834:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573762.371836:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573762.371839:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573762.371841:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164608, portal 4 00000100:00000001:0.0:1788573762.371842:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573762.371843:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287353728 : -109714422197888 : ffff9c371ed69180) 00000100:00000040:0.0:1788573762.371847:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371852:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573762.371856:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573762.371858:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573762.371860:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810164608, offset 0 00000400:00000200:0.0:1788573762.371862:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371865:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371868:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573762.371870:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573762.371872:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573762.371873:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573762.371875:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573762.371877:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541376:541376:256:4294967295] 192.168.202.148@tcp LPNI seq info [541341:541341:8:4294967295] 00000400:00000200:0.0:1788573762.371880:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.371884:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573762.371888:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371890:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573762.371893:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573762.371895:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573762.371902:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.371903:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573762.371905:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573762.371907:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573762.371909:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573762.371911:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573762.371912:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573762.371915:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371920:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.371922:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371926:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371928:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573762.371933:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573762.371935:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788573762.371938:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573762.371939:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308901538 00000100:00000001:0.0:1788573762.371941:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371942:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308901538 00000100:00000001:0.0:1788573762.371944:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.371945:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573762.371946:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.371947:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573762.371950:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573762/real 1788573762] req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573762.371955:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573762.371956:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573762.371957:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573762.371958:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573762.371960:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573762.371961:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573762.371963:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573762.371968:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371979:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573762.371980:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.371982:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.371983:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573762.371985:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.371990:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573762.371992:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372000:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573762.372002:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372003:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.372004:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372005:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573762.372007:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573762.372008:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573762.372010:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573762.372012:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573762.372014:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573762.372019:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164608:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573762.372022:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372023:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.372025:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed68000 x1875440810164352/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372029:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573762.372030:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573762.372032:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788573762.372034:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573762.372035:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573762.372036:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573762.372038:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573762.372039:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da34c00. 02000000:00000001:0.0:1788573762.372041:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573762.372043:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573762.372044:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed68000. 00000100:00000001:0.0:1788573762.372046:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573762.372047:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372051:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.372053:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69c00 x1875440810164480/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372058:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573762.372058:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573762.372059:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788573762.372061:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573762.372061:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573762.372062:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788573762.372063:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573762.372064:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37c00. 02000000:00000001:0.0:1788573762.372066:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573762.372066:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573762.372067:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69c00. 00000100:00000001:0.0:1788573762.372069:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573762.372069:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372070:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.372072:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed69180 x1875440810164608/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372076:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573762.372076:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573762.372077:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573762.372078:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573762.372079:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573762.372080:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788573762.372081:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573762.372082:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da36400. 02000000:00000001:0.0:1788573762.372083:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573762.372084:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573762.372084:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed69180. 00000100:00000001:0.0:1788573762.372086:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573762.372086:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372088:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372091:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573762.372092:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.372094:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573762.372098:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573762.372099:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573762.372102:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372107:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.372108:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.372109:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573762.372111:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573762.372112:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.372114:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573762.372115:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573762.372117:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573762.372118:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573762.372119:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573762.372120:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573762.372122:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573762.372124:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788573762.372126:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788573762.372128:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788573762.372130:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810164736, portal 4 00000100:00000001:0.0:1788573762.372132:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573762.372133:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359287360896 : -109714422190720 : ffff9c371ed6ad80) 00000100:00000040:0.0:1788573762.372137:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372157:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573762.372158:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788573762.372160:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788573762.372162:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810164736, offset 0 00000400:00000200:0.0:1788573762.372165:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788573762.372168:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.372171:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788573762.372173:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573762.372174:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788573762.372180:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573762.372182:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788573762.372184:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [541377:541377:256:4294967295] 192.168.202.148@tcp LPNI seq info [541342:541342:8:4294967295] 00000400:00000200:0.0:1788573762.372188:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573762.372195:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788573762.372200:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.372203:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788573762.372206:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788573762.372209:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573762.372221:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788573762.372223:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788573762.372226:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573762.372229:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573762.372233:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788573762.372235:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788573762.372238:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573762.372242:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372250:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.372253:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372259:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.372260:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573762.372263:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573762.372265:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788573762.372268:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788573762.372269:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4308901547 00000100:00000001:0.0:1788573762.372271:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.372272:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4308901547 00000100:00000001:0.0:1788573762.372279:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4308905182 00000100:00000001:0.0:1788573762.372280:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573762.372282:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.372283:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573762.372286:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573762/real 1788573762] req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573762.372292:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573762.372292:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573762.372294:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788573762.372295:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788573762.372296:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788573762.372298:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573762.372300:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573762.372306:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372310:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573762.372310:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372311:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.372312:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573762.372314:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372318:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573762.372321:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372325:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573762.372326:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372327:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573762.372328:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372332:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573762.372333:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573762.372335:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573762.372338:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788573762.372339:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573762.372342:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573762.372347:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ed6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810164736:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788573762.372350:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372351:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573762.372353:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ed6ad80 x1875440810164736/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573762.372356:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573762.372357:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573762.372359:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c800. 02000000:00000001:0.0:1788573762.372360:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573762.372361:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573762.372362:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788573762.372363:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573762.372364:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c372da37000. 02000000:00000001:0.0:1788573762.372365:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573762.372366:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573762.372367:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ed6ad80. 00000100:00000001:0.0:1788573762.372369:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573762.372370:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372371:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.372373:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573762.372374:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573762.372376:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573762.372377:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573763.777804:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573763.777807:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573763.777816:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573766.209735:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308905182, at 4308905216 00000100:00000010:0.0:1788573766.209753:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000020:00000001:0.0:1788573768.897951:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573768.897960:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573768.897965:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573772.289750:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308911263, at 4308911296 00000100:00000010:0.0:1788573772.289770:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788573774.017826:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573774.017830:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573774.017832:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573777.473725:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308916446, at 4308916480 00000100:00000010:0.0:1788573777.473737:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788573779.137858:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573779.137862:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573779.137866:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573782.593717:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308921566, at 4308921600 00000100:00000010:0.0:1788573782.593732:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788573784.257846:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573784.257850:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573784.257852:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573787.713755:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308926687, at 4308926720 00000100:00000010:0.0:1788573787.713774:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 10000000:00000001:0.0:1788573788.481843:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573788.481864:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9ec0. 00000020:00000010:0.0:1788573788.481885:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788573788.481893:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788573788.481898:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788573788.481905:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573788.481911:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788573788.481925:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573788.481932:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ac00. 20000000:00000010:0.0:1788573788.481958:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788573788.481965:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573788.481968:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573788.481976:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788573788.481986:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788573788.481990:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573788.481995:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788573788.481999:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573788.482003:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ac00. 20000000:00000010:0.0:1788573788.482007:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788573788.482010:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788573788.482013:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9ec0. 10000000:01000000:0.0:1788573788.482017:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573788.482022:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573788.487940:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573788.487944:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573788.487947:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573788.487951:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573788.487953:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573788.487960:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573788.487963:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573788.487966:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573788.487968:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573788.487970:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573788.487973:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788573789.377810:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573789.377815:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573789.377819:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573790.017766:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573790.017773:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573790.017778:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573790.017780:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573790.017782:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573790.017783:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573790.017785:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573790.017787:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573790.017804:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573790.017806:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573790.017807:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573790.017814:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573790.017818:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573790.017823:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573790.017826:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573790.017832:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573790.017839:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788573790.017845:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573790.017848:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573790.017851:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573790.017857:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788573790.017861:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573790.017863:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31800. 00000020:00000001:0.0:1788573790.017867:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573790.017869:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31800 with handle 0x1441586093168b4c to hash 00000020:00000001:0.0:1788573790.017871:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573790.017872:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788573790.017875:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788573790.017878:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573790.017881:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573790.017884:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573790.017892:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573790.017902:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573790.017907:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573790.017908:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573790.017913:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573790.017918:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788573790.017921:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573790.017922:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788573790.017926:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573790.017930:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573790.017931:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573790.017933:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573790.017942:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573790.017949:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573790.017952:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573790.017957:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573790.017960:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573790.017965:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573790.017967:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573790.017968:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573790.017977:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788573790.017979:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573790.017981:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573790.017983:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573790.017985:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573790.017987:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573790.017990:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573790.017996:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810168832:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573790.018000:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573790.018002:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573790.018004:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573790.018009:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573790.018012:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573790.018019:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573790.018021:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573790.018026:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573790.018028:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573790.018030:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573790.018034:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573790.018036:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573790.018038:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=4 00000020:00000001:0.0:1788573790.018041:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788573790.018044:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00010000:0.0:1788573790.018049:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018056:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573790.018061:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018068:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573790.018071:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018077:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573790.018078:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573790.018080:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018084:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573790.018085:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573790.018087:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573790.018094:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573790.018099:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573790.018101:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573790.018103:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018107:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573790.018108:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573790.018109:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573790.018110:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=5 00000020:00000001:0.0:1788573790.018111:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788573790.018113:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788573790.018115:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573790.018117:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573790.018121:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573790.018122:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573790.018126:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573790.018128:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573790.018129:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573790.018130:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573790.018132:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573790.018134:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573790.018135:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573790.018137:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573790.018138:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573790.018140:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31800 with handle 0x1441586093168b4c from hash 00010000:00000001:0.0:1788573790.018158:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573790.018162:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573790.018171:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573790.018172:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573790.018174:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573790.018176:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573790.018178:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573790.018179:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573790.018180:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573790.018181:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573790.018184:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573790.018188:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573790.018189:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573790.018190:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573790.018191:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573790.018193:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573790.018194:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573790.018195:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573790.018197:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168b4c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573790.018202:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788573790.018226:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573790.018227:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573790.018229:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573790.018233:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810168832/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573790.018242:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573790.018244:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573790.018246:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573790.018247:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573790.018250:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788573790.018253:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573790.018255:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573790.018256:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788573790.018264:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573790.018265:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573790.018267:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573790.018268:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573790.018271:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573793.857744:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308932831, at 4308932864 00000100:00000010:0.0:1788573793.857759:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000020:00000001:0.0:1788573794.497804:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573794.497809:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573794.497813:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573795.521847:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573795.521853:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573795.521857:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573796.609949:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788573798.977748:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308937950, at 4308937984 00000100:00000010:0.0:1788573798.977766:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000020:00000001:0.0:1788573799.617865:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573799.617869:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573799.617872:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573800.641838:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573800.641844:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573800.641848:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573804.097716:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308943071, at 4308943104 00000100:00000010:0.0:1788573804.097730:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788573804.737808:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573804.737811:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573804.737814:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573805.761876:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573805.761880:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573805.761883:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573809.217729:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308948190, at 4308948224 00000100:00000010:0.0:1788573809.217744:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788573809.857823:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573809.857826:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573809.857846:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573810.881797:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573810.881800:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573810.881803:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573814.977845:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573814.977849:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573814.977853:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573816.001777:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573816.001781:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573816.001783:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573816.193703:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308955167, at 4308955200 00000100:00000010:0.0:1788573816.193719:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788573820.097863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573820.097866:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573820.097869:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573821.121832:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573821.121837:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573821.121840:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573821.505752:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308960478, at 4308960512 00000100:00000010:0.0:1788573821.505767:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788573825.217825:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573825.217829:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573825.217834:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573826.241828:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573826.241831:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573826.241834:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573826.625752:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308965598, at 4308965632 00000100:00000010:0.0:1788573826.625767:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788573830.337791:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573830.337796:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573830.337800:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573831.361793:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573831.361798:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573831.361803:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573831.745733:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308970718, at 4308970752 00000100:00000010:0.0:1788573831.745748:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788573832.385827:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573832.385833:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573832.385837:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573834.498012:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788573835.457792:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573835.457796:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573835.457799:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573836.481839:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573836.481843:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573836.481846:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573837.441784:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308976415, at 4308976448 00000100:00000010:0.0:1788573837.441819:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215660. 00000020:00000001:0.0:1788573837.505851:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573837.505857:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573837.505861:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573840.577836:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573840.577840:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573840.577842:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573841.601866:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573841.601869:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573841.601872:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573842.497749:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308981470, at 4308981504 00000100:00000010:0.0:1788573842.497766:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788573842.625836:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573842.625839:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573842.625842:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573843.777791:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573843.777802:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573843.777804:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573843.777808:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573843.777812:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573843.777834:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14310 00000400:00000200:0.0:1788573843.777843:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573843.777849:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573843.777852:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573843.777855:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.777858:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.777861:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.777864:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573843.777868:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573843.777870:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573843.777873:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573843.777877:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573843.777880:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573843.777882:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573843.777885:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.777887:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.777889:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.777891:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573843.777894:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573843.777898:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573843.777901:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573843.777904:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573843.777909:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573843.777912:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.777914:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573843.777917:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573843.777923:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573843.777927:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573843.777931:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573843.777933:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.777936:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573843.777939:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573843.777949:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573843.777958:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573843.777965:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.777976:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810176896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573843.777988:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573843.777992:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.777997:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810176896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573843.778025:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573843.778028:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573843.778032:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573843.778033:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573843.778035:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573843.778038:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573843.778041:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14310 00000400:00000200:0.0:1788573843.778046:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573843.778050:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573843.778053:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573843.778056:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778060:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778065:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778069:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573843.778075:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573843.778079:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573843.778082:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573843.778086:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573843.778092:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573843.778098:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573843.778107:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778119:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778126:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778130:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573843.778133:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573843.778137:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778141:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573843.778143:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573843.778149:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573843.778155:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778159:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573843.778167:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573843.778175:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778179:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573843.778186:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573843.778190:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573843.778196:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573843.778199:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778202:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573843.778207:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573843.778213:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573843.778222:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573843.778229:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.778241:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810177024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573843.778255:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573843.778259:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.778266:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810177024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573843.778280:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573843.778283:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573843.778288:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573843.778298:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573843.778301:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573843.778307:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573843.778312:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14310 00000400:00000200:0.0:1788573843.778338:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573843.778345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573843.778349:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573843.778353:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778358:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778364:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778369:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573843.778372:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573843.778374:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573843.778377:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573843.778381:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573843.778384:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573843.778388:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573843.778392:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778394:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778398:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778400:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573843.778404:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573843.778407:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778410:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573843.778413:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573843.778418:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573843.778422:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778424:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573843.778426:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573843.778430:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778439:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573843.778444:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573843.778449:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573843.778454:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573843.778456:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778459:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573843.778461:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573843.778464:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573843.778470:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573843.778474:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.778491:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810177152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573843.778502:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573843.778506:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.778510:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810177152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573843.778519:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573843.778525:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573843.778527:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573843.778529:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573843.778532:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573843.778535:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573843.778540:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14310 00000400:00000200:0.0:1788573843.778547:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573843.778552:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573843.778556:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573843.778561:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778564:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778569:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778574:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573843.778578:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573843.778585:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573843.778590:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573843.778592:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573843.778596:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573843.778600:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573843.778605:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778606:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778608:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778611:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573843.778613:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573843.778617:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778621:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573843.778625:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573843.778628:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573843.778632:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778657:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573843.778659:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573843.778661:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778662:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573843.778665:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573843.778667:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573843.778669:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573843.778670:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778673:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573843.778674:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573843.778677:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573843.778698:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573843.778703:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.778711:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810177280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573843.778722:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573843.778726:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.778740:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810177280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573843.778752:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573843.778754:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573843.778761:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573843.778765:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573843.778767:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573843.778774:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573843.778781:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14310 00000400:00000200:0.0:1788573843.778790:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573843.778795:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573843.778800:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573843.778806:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778810:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778811:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778813:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573843.778815:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573843.778816:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573843.778819:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573843.778822:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573843.778825:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573843.778829:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573843.778832:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778837:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573843.778841:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778846:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573843.778849:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573843.778853:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573843.778860:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573843.778867:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573843.778872:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573843.778878:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778883:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573843.778885:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573843.778887:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778891:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573843.778899:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573843.778902:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573843.778905:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573843.778908:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.778913:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573843.778915:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573843.778918:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573843.778923:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573843.778929:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.778936:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810177408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573843.778947:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573843.778950:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.778956:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810177408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573843.778965:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.778972:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14320) 00000020:00000001:0.0:1788573845.697828:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573845.697833:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573845.697836:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573846.721856:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573846.721860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573846.721862:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573847.617734:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308986590, at 4308986624 00000100:00000010:0.0:1788573847.617748:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00080000:0.0:1788573848.897724:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573848.897753:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573848.897756:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573848.897760:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573848.897764:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573848.897771:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14315 00000400:00000200:0.0:1788573848.897778:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573848.897784:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573848.897786:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573848.897789:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897791:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.897793:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897795:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573848.897798:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573848.897799:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573848.897801:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573848.897804:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573848.897805:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573848.897807:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573848.897809:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897811:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.897812:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897813:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573848.897815:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573848.897818:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897819:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573848.897822:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573848.897824:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573848.897826:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.897828:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573848.897830:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573848.897836:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573848.897843:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573848.897846:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573848.897847:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.897850:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573848.897852:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573848.897856:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573848.897863:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573848.897868:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.897875:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810177536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573848.897883:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573848.897886:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.897890:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810177536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573848.897909:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573848.897912:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573848.897914:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573848.897915:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573848.897916:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573848.897918:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573848.897920:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14315 00000400:00000200:0.0:1788573848.897922:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573848.897924:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573848.897926:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573848.897928:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897930:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.897931:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897933:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573848.897935:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573848.897936:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573848.897937:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573848.897944:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573848.897949:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573848.897951:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573848.897957:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897963:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.897968:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897974:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573848.897979:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573848.897983:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573848.897989:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573848.897994:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573848.898003:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573848.898009:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898014:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573848.898021:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573848.898027:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898033:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573848.898040:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573848.898043:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573848.898049:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573848.898053:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898056:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573848.898057:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573848.898060:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573848.898075:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573848.898082:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.898092:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810177664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573848.898105:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573848.898108:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.898114:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810177664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573848.898134:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573848.898138:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573848.898142:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573848.898144:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573848.898146:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573848.898149:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573848.898154:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14315 00000400:00000200:0.0:1788573848.898211:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573848.898217:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573848.898219:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573848.898224:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898226:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898231:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898235:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573848.898238:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573848.898240:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573848.898242:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573848.898244:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573848.898247:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573848.898251:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573848.898257:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898261:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898267:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898272:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573848.898277:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573848.898283:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898290:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573848.898301:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573848.898307:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573848.898312:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898315:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573848.898319:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573848.898324:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898335:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573848.898341:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573848.898345:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573848.898350:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573848.898352:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898353:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573848.898355:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573848.898358:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573848.898364:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573848.898369:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.898377:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810177792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573848.898386:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573848.898389:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.898395:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810177792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573848.898407:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573848.898410:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573848.898415:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573848.898418:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573848.898422:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573848.898426:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573848.898431:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14315 00000400:00000200:0.0:1788573848.898435:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573848.898440:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573848.898444:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573848.898455:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898459:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898463:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898467:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573848.898472:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573848.898475:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573848.898476:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573848.898480:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573848.898484:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573848.898488:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573848.898494:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898498:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898501:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898506:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573848.898511:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573848.898515:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898519:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573848.898523:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573848.898527:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573848.898530:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898535:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573848.898538:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573848.898543:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898547:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573848.898551:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573848.898556:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573848.898561:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573848.898564:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898568:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573848.898572:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573848.898574:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573848.898594:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573848.898606:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.898612:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810177920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573848.898618:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573848.898622:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.898632:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810177920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573848.898659:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573848.898662:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573848.898665:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573848.898666:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573848.898668:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573848.898670:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573848.898673:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14315 00000400:00000200:0.0:1788573848.898676:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573848.898678:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573848.898680:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573848.898685:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898690:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898694:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898697:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573848.898701:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573848.898704:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573848.898706:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573848.898707:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573848.898709:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573848.898710:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573848.898716:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898722:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573848.898736:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898742:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573848.898747:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573848.898754:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573848.898760:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573848.898765:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573848.898769:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573848.898772:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898774:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573848.898779:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573848.898781:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898784:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573848.898790:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573848.898794:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573848.898799:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573848.898801:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.898802:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573848.898804:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573848.898808:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573848.898814:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573848.898820:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.898827:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810178048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573848.898842:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573848.898846:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.898853:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810178048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573848.898861:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.898867:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14325) 00000020:00000001:0.0:1788573850.817799:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573850.817802:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573850.817805:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573851.841799:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573851.841803:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573851.841806:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573852.737768:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308991710, at 4308991744 00000100:00000010:0.0:1788573852.737783:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788573854.017764:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573854.017779:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573854.017783:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573854.017789:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573854.017793:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573854.017799:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14320 00000400:00000200:0.0:1788573854.017817:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573854.017826:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573854.017830:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573854.017835:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.017846:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.017852:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.017857:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573854.017864:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573854.017869:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573854.017875:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573854.017884:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573854.017886:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573854.017895:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573854.017903:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.017908:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.017911:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.017920:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573854.017931:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573854.017939:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573854.017973:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573854.017983:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573854.017990:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573854.017997:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018000:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573854.018002:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573854.018015:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573854.018020:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573854.018024:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573854.018025:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018029:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573854.018031:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573854.018036:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573854.018042:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573854.018050:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.018059:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810178176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573854.018067:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573854.018071:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.018075:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810178176/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573854.018104:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573854.018108:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573854.018112:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573854.018113:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573854.018115:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573854.018118:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573854.018121:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14320 00000400:00000200:0.0:1788573854.018129:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573854.018135:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573854.018138:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573854.018142:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018157:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.018166:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018173:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573854.018198:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573854.018203:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573854.018209:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573854.018216:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573854.018223:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573854.018234:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573854.018245:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018251:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.018261:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018270:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573854.018276:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573854.018283:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018291:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573854.018299:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573854.018310:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573854.018319:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018334:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573854.018345:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573854.018353:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018358:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573854.018364:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573854.018372:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573854.018379:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573854.018383:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018396:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573854.018400:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573854.018408:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573854.018423:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573854.018431:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.018454:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810178304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573854.018472:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573854.018494:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.018501:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810178304/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573854.018531:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573854.018540:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573854.018553:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573854.018557:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573854.018562:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573854.018572:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573854.018585:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14320 00000400:00000200:0.0:1788573854.018598:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573854.018605:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573854.018612:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573854.018623:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018629:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.018660:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018664:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573854.018667:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573854.018669:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573854.018671:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573854.018673:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573854.018676:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573854.018679:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573854.018683:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018685:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.018688:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018703:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573854.018713:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573854.018719:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018725:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573854.018728:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573854.018731:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573854.018733:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018735:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573854.018741:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573854.018743:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018745:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573854.018749:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573854.018752:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573854.018757:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573854.018759:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.018766:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573854.018772:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573854.018775:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573854.018783:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573854.018788:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.018798:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810178432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573854.018806:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573854.018809:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.018813:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810178432/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573854.018830:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573854.018841:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573854.018851:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573854.018857:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573854.018861:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573854.018877:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573854.018884:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14320 00000400:00000200:0.0:1788573854.018903:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573854.018911:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573854.018919:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573854.018926:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018932:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.018942:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.018948:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573854.018958:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573854.018965:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573854.018972:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573854.018978:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573854.018985:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573854.018994:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573854.019010:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019017:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.019025:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019036:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573854.019042:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573854.019058:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019068:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573854.019073:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573854.019081:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573854.019085:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019091:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573854.019095:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573854.019104:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019112:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573854.019121:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573854.019131:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573854.019136:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573854.019139:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019147:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573854.019152:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573854.019162:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573854.019171:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573854.019176:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.019184:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810178560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573854.019205:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573854.019209:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.019217:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810178560/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573854.019266:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573854.019272:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573854.019276:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573854.019278:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573854.019282:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573854.019289:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573854.019294:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14320 00000400:00000200:0.0:1788573854.019302:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573854.019307:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573854.019315:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573854.019329:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019336:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.019345:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019353:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573854.019361:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573854.019368:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573854.019376:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573854.019385:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573854.019392:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573854.019395:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573854.019401:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019404:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573854.019408:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019417:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573854.019420:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573854.019429:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573854.019439:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573854.019440:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573854.019444:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573854.019449:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019457:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573854.019459:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573854.019468:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019471:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573854.019493:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573854.019497:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573854.019499:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573854.019500:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573854.019502:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573854.019504:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573854.019511:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573854.019514:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573854.019518:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.019525:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810178688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573854.019542:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573854.019546:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.019557:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810178688/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573854.019601:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573854.019617:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14330) 00000100:00080000:0.0:1788573855.810212:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573855.810218:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573855.810221:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573855.810224:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573855.810228:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573855.810233:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14325 00000400:00000200:0.0:1788573855.810241:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573855.810246:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573855.810250:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573855.810253:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810256:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810259:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810261:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573855.810264:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573855.810266:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573855.810268:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573855.810270:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573855.810273:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573855.810275:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573855.810278:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810280:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810281:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810283:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573855.810286:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573855.810289:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810292:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573855.810304:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573855.810308:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573855.810310:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810313:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573855.810315:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573855.810320:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573855.810323:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573855.810326:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573855.810327:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810330:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573855.810332:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573855.810334:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573855.810339:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573855.810343:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.810350:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810178944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573855.810370:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573855.810372:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.810377:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810178944/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573855.810392:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573855.810395:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573855.810398:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573855.810399:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573855.810400:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573855.810402:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573855.810404:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14325 00000400:00000200:0.0:1788573855.810407:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573855.810409:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573855.810411:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573855.810413:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810416:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810423:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810426:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573855.810429:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573855.810432:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573855.810434:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573855.810449:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573855.810452:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573855.810457:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573855.810465:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810470:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810476:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810482:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573855.810487:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573855.810496:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810500:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573855.810504:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573855.810513:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573855.810519:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810523:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573855.810530:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573855.810534:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810536:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573855.810544:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573855.810549:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573855.810554:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573855.810557:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810558:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573855.810560:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573855.810563:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573855.810570:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573855.810577:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.810587:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810179072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573855.810611:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573855.810617:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.810626:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810179072/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573855.810676:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573855.810680:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573855.810685:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573855.810687:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573855.810689:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573855.810694:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573855.810698:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14325 00000400:00000200:0.0:1788573855.810756:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573855.810762:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573855.810765:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573855.810768:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810771:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810775:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810779:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573855.810781:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573855.810783:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573855.810786:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573855.810789:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573855.810792:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573855.810796:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573855.810802:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810807:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.810809:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810813:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573855.810823:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573855.810829:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573855.810834:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573855.810838:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573855.810882:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573855.810886:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810889:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573855.810894:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573855.810903:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810909:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573855.810915:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573855.810921:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573855.810926:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573855.810929:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.810934:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573855.810939:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573855.810943:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573855.810950:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573855.810955:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.810965:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810179200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573855.810977:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573855.810982:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.810997:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810179200/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573855.811021:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573855.811023:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573855.811026:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573855.811027:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573855.811029:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573855.811035:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573855.811061:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14325 00000400:00000200:0.0:1788573855.811069:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573855.811075:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573855.811079:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573855.811087:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811092:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.811097:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811100:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573855.811104:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573855.811106:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573855.811107:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573855.811109:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573855.811110:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573855.811112:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573855.811121:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811125:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.811130:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811135:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573855.811140:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573855.811146:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811149:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573855.811151:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573855.811156:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573855.811161:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811165:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573855.811169:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573855.811173:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811178:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573855.811182:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573855.811186:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573855.811188:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573855.811196:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811198:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573855.811200:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573855.811203:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573855.811207:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573855.811212:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.811219:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810179328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573855.811230:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573855.811234:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.811240:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810179328/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573855.811252:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573855.811256:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573855.811262:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573855.811263:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573855.811264:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573855.811266:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573855.811269:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14325 00000400:00000200:0.0:1788573855.811275:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573855.811280:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573855.811282:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573855.811288:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811291:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.811294:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811299:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573855.811303:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573855.811305:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573855.811308:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573855.811310:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573855.811313:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573855.811323:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573855.811329:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811332:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573855.811337:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811342:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573855.811346:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573855.811351:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573855.811355:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573855.811358:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573855.811362:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573855.811365:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811368:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573855.811371:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573855.811377:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811382:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573855.811385:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573855.811389:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573855.811391:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573855.811392:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.811393:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573855.811395:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573855.811398:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573855.811404:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573855.811410:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.811419:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810179456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573855.811433:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573855.811436:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.811458:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810179456/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573855.811486:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573855.811500:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14332) 00000020:00000001:0.0:1788573855.937870:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573855.937874:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573855.937876:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573856.961790:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573856.961795:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573856.961798:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573859.649737:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4308998623, at 4308998656 00000100:00000010:0.0:1788573859.649754:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788573861.057828:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573861.057833:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573861.057836:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573861.185751:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573861.185760:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573861.185762:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573861.185765:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573861.185769:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573861.185775:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14327 00000400:00000200:0.0:1788573861.185783:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573861.185788:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573861.185791:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573861.185793:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185795:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.185797:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185799:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573861.185802:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573861.185804:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573861.185806:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573861.185809:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573861.185811:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573861.185812:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573861.185814:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185833:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.185834:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185836:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573861.185838:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573861.185840:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185842:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573861.185844:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573861.185847:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573861.185849:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.185851:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573861.185853:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573861.185858:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573861.185860:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573861.185864:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573861.185865:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.185867:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573861.185869:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573861.185874:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573861.185880:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573861.185885:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.185892:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810179584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573861.185900:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573861.185904:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.185908:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810179584/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573861.185927:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573861.185930:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573861.185932:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573861.185934:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573861.185935:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573861.185939:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573861.185947:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14327 00000400:00000200:0.0:1788573861.185952:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573861.185956:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573861.185959:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573861.185963:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185965:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.185969:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.185974:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573861.185979:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573861.185983:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573861.185986:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573861.185990:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573861.185995:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573861.185999:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573861.186004:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186009:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186012:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186016:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573861.186023:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573861.186029:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186033:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573861.186039:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573861.186047:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573861.186055:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186060:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573861.186065:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573861.186074:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186077:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573861.186084:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573861.186089:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573861.186095:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573861.186102:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186105:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573861.186108:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573861.186112:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573861.186119:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573861.186124:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.186132:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810179712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573861.186146:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573861.186149:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.186155:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810179712/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573861.186165:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573861.186169:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573861.186174:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573861.186175:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573861.186177:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573861.186183:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573861.186188:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14327 00000400:00000200:0.0:1788573861.186192:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573861.186194:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573861.186196:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573861.186201:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186206:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186208:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186210:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573861.186214:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573861.186217:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573861.186221:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573861.186230:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573861.186233:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573861.186244:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573861.186250:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186256:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186263:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186268:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573861.186274:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573861.186278:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186281:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573861.186285:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573861.186290:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573861.186293:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186298:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573861.186302:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573861.186305:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186310:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573861.186313:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573861.186316:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573861.186319:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573861.186322:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186325:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573861.186342:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573861.186345:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573861.186353:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573861.186359:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.186366:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810179840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573861.186387:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573861.186393:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.186405:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810179840/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573861.186423:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573861.186431:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573861.186436:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573861.186439:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573861.186443:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573861.186448:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573861.186453:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14327 00000400:00000200:0.0:1788573861.186461:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573861.186465:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573861.186468:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573861.186472:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186474:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186484:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186487:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573861.186491:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573861.186492:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573861.186496:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573861.186499:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573861.186501:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573861.186507:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573861.186510:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186514:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186519:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186520:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573861.186524:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573861.186528:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186533:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573861.186537:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573861.186540:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573861.186545:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186547:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573861.186556:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573861.186561:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186566:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573861.186571:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573861.186575:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573861.186580:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573861.186582:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186586:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573861.186588:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573861.186592:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573861.186598:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573861.186604:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.186613:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810179968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573861.186626:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573861.186630:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.186661:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810179968/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573861.186675:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573861.186681:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573861.186689:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573861.186692:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573861.186695:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573861.186697:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573861.186700:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14327 00000400:00000200:0.0:1788573861.186707:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573861.186713:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573861.186718:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573861.186724:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186727:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186729:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186736:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573861.186741:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573861.186743:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573861.186745:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573861.186749:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573861.186754:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573861.186758:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573861.186765:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186766:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573861.186770:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186774:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573861.186777:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573861.186780:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573861.186785:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573861.186789:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573861.186792:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573861.186794:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186798:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573861.186801:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573861.186806:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186811:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573861.186815:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573861.186819:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573861.186821:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573861.186824:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573861.186825:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573861.186827:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573861.186830:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573861.186835:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573861.186841:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.186850:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810180096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573861.186875:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573861.186878:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.186884:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810180096/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573861.186899:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573861.186906:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14337) 00000020:00000001:0.0:1788573862.081858:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573862.081862:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573862.081865:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573865.025720:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309003998, at 4309004032 00000100:00000010:0.0:1788573865.025734:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788573866.177809:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573866.177813:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573866.177816:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573867.201825:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573867.201829:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573867.201832:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573870.145742:0:267687:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309009118, at 4309009152 00000100:00000010:0.0:1788573870.145761:0:267687:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788573871.297873:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573871.297880:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573871.297886:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573872.321812:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573872.321815:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573872.321818:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573873.409961:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573873.409966:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573873.409976:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573873.409980:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573873.409982:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573875.265749:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309014238, at 4309014272 00000100:00000010:0.0:1788573875.265766:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788573876.417812:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573876.417816:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573876.417819:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573876.545759:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573876.545767:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573876.545770:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573876.545773:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573876.545776:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573876.545783:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14343 00000400:00000200:0.0:1788573876.545791:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573876.545797:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573876.545799:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573876.545802:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545804:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.545806:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545808:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573876.545811:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573876.545813:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573876.545815:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573876.545818:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573876.545819:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573876.545821:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573876.545823:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545824:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.545825:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545827:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573876.545829:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573876.545832:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545834:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573876.545836:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573876.545853:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573876.545856:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.545858:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573876.545860:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573876.545867:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573876.545870:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573876.545873:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573876.545875:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.545878:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573876.545880:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573876.545884:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573876.545891:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573876.545896:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.545904:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810181504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573876.545912:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573876.545916:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.545920:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810181504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573876.545941:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573876.545944:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573876.545946:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573876.545948:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573876.545949:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573876.545950:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573876.545952:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14343 00000400:00000200:0.0:1788573876.545955:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573876.545957:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573876.545959:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573876.545961:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545963:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.545969:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.545971:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573876.545973:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573876.545975:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573876.545976:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573876.545980:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573876.545983:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573876.545988:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573876.545996:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546000:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546006:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546011:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573876.546028:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573876.546035:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546040:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573876.546042:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573876.546048:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573876.546052:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546054:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573876.546060:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573876.546067:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546071:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573876.546078:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573876.546083:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573876.546086:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573876.546089:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546092:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573876.546096:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573876.546097:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573876.546102:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573876.546110:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.546120:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810181632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573876.546136:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573876.546139:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.546146:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810181632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573876.546162:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573876.546166:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573876.546169:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573876.546170:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573876.546172:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573876.546175:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573876.546178:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14343 00000400:00000200:0.0:1788573876.546182:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573876.546188:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573876.546190:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573876.546193:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546194:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546196:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546201:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573876.546202:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573876.546204:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573876.546206:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573876.546209:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573876.546211:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573876.546213:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573876.546220:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546222:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546225:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546229:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573876.546240:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573876.546246:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546249:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573876.546252:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573876.546255:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573876.546259:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546260:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573876.546264:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573876.546268:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546270:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573876.546274:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573876.546278:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573876.546281:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573876.546284:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546288:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573876.546290:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573876.546293:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573876.546299:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573876.546303:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.546312:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810181760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573876.546319:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573876.546321:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.546334:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810181760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573876.546346:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573876.546351:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573876.546354:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573876.546357:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573876.546359:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573876.546361:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573876.546369:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14343 00000400:00000200:0.0:1788573876.546376:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573876.546381:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573876.546385:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573876.546391:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546395:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546400:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546403:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573876.546406:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573876.546409:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573876.546412:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573876.546414:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573876.546418:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573876.546420:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573876.546423:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546428:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546432:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546436:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573876.546442:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573876.546446:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546452:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573876.546456:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573876.546459:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573876.546461:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546465:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573876.546467:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573876.546470:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546471:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573876.546476:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573876.546480:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573876.546485:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573876.546490:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546492:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573876.546496:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573876.546498:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573876.546502:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573876.546508:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.546521:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810181888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573876.546528:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573876.546530:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.546535:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810181888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573876.546542:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573876.546546:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573876.546553:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573876.546556:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573876.546558:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573876.546561:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573876.546565:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14343 00000400:00000200:0.0:1788573876.546570:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573876.546575:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573876.546576:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573876.546578:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546580:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546585:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546586:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573876.546588:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573876.546589:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573876.546591:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573876.546592:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573876.546599:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573876.546607:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573876.546613:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546616:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573876.546621:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546624:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573876.546629:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573876.546632:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573876.546671:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573876.546674:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573876.546677:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573876.546682:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546687:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573876.546690:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573876.546694:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546699:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573876.546703:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573876.546708:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573876.546714:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573876.546715:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573876.546720:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573876.546721:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573876.546723:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573876.546730:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573876.546735:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.546746:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810182016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573876.546771:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573876.546774:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.546784:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810182016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573876.546802:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573876.546814:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14353) 00000020:00000001:0.0:1788573877.441791:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573877.441794:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573877.441797:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573877.442180:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573877.442184:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573877.442186:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573877.442189:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573877.442192:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573877.442195:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14348 00000400:00000200:0.0:1788573877.442202:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573877.442206:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573877.442209:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573877.442211:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442213:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442215:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442217:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573877.442220:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573877.442221:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573877.442222:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573877.442224:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573877.442225:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573877.442226:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573877.442228:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442230:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442231:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442233:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573877.442235:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573877.442237:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442239:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573877.442246:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573877.442249:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573877.442251:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442252:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573877.442254:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573877.442257:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573877.442259:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573877.442261:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573877.442263:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442264:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573877.442266:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573877.442268:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573877.442271:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573877.442274:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.442279:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810182272/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573877.442285:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573877.442287:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.442290:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810182272/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573877.442310:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573877.442312:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573877.442314:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573877.442319:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573877.442321:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573877.442323:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573877.442325:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14348 00000400:00000200:0.0:1788573877.442328:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573877.442330:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573877.442332:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573877.442334:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442336:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442342:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442343:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573877.442345:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573877.442346:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573877.442348:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573877.442350:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573877.442354:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573877.442356:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573877.442361:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442364:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442367:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442370:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573877.442374:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573877.442379:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442384:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573877.442388:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573877.442394:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573877.442397:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442401:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573877.442406:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573877.442411:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442414:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573877.442420:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573877.442424:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573877.442428:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573877.442429:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442433:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573877.442436:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573877.442437:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573877.442442:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788573877.442448:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.442458:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810182400/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573877.442482:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573877.442486:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.442492:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810182400/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573877.442501:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573877.442504:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573877.442509:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573877.442511:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573877.442514:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573877.442519:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573877.442522:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14348 00000400:00000200:0.0:1788573877.442529:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573877.442535:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573877.442537:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573877.442542:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442544:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442549:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442551:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573877.442556:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573877.442559:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573877.442560:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573877.442564:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573877.442569:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573877.442572:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573877.442576:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442579:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442583:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442585:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573877.442593:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573877.442600:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442605:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573877.442609:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573877.442613:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573877.442615:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442619:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573877.442620:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573877.442624:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442626:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573877.442631:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573877.442657:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573877.442660:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573877.442661:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442663:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573877.442664:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573877.442666:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573877.442670:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788573877.442675:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.442682:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810182528/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573877.442693:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573877.442697:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.442703:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810182528/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573877.442710:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573877.442713:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573877.442717:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573877.442720:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573877.442722:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573877.442726:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573877.442735:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14348 00000400:00000200:0.0:1788573877.442738:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573877.442741:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573877.442742:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573877.442747:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442748:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442751:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442753:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573877.442756:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573877.442759:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573877.442760:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573877.442763:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573877.442767:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573877.442769:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573877.442773:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442774:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442775:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442778:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573877.442781:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573877.442786:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442791:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573877.442797:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573877.442800:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573877.442805:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442807:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573877.442811:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573877.442814:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442816:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573877.442819:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573877.442824:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573877.442827:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573877.442835:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.442837:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573877.442838:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573877.442839:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573877.442845:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788573877.442850:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.442858:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810182656/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573877.442866:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573877.442870:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.442876:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810182656/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573877.442902:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573877.442906:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573877.442913:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573877.442914:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573877.442917:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573877.442920:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573877.442922:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14348 00000400:00000200:0.0:1788573877.442925:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573877.442931:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573877.442933:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573877.442938:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442943:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442945:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442950:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573877.442952:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573877.442953:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573877.442954:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573877.442956:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573877.442961:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573877.442969:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573877.442976:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442981:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573877.442985:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442988:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573877.442990:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573877.442995:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573877.442997:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573877.443012:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573877.443016:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573877.443021:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.443023:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573877.443025:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573877.443028:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.443032:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573877.443038:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573877.443043:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573877.443049:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573877.443051:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.443054:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573877.443056:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573877.443058:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573877.443064:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573877.443070:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.443078:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810182784/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573877.443094:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573877.443098:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.443105:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810182784/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573877.443126:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573877.443141:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14354) 00000100:00000040:0.0:1788573881.281745:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309020255, at 4309020288 00000100:00000010:0.0:1788573881.281758:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788573881.537848:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573881.537863:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573881.537868:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573882.561839:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573882.561843:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573882.561846:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573886.529773:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309025502, at 4309025536 00000100:00000010:0.0:1788573886.529792:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788573886.657858:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573886.657862:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573886.657866:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573887.681901:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573887.681905:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573887.681909:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573891.649753:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309030623, at 4309030656 00000100:00000010:0.0:1788573891.649766:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788573891.777851:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573891.777856:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573891.777860:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573892.801835:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573892.801840:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573892.801844:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573896.769750:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309035743, at 4309035776 00000100:00000010:0.0:1788573896.769775:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788573896.897807:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573896.897811:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573896.897815:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573897.921775:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573897.921780:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573897.921785:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573899.202279:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573899.202292:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573899.202293:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573899.202296:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573899.202299:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573899.202303:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14369 00000400:00000200:0.0:1788573899.202308:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573899.202313:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573899.202316:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573899.202319:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202321:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202323:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202325:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573899.202327:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573899.202328:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573899.202329:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573899.202331:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573899.202333:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573899.202335:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573899.202337:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202338:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202340:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202341:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573899.202343:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573899.202345:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202347:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573899.202349:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573899.202352:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573899.202353:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202355:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573899.202356:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573899.202360:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573899.202365:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573899.202367:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573899.202369:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202371:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573899.202372:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573899.202374:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573899.202378:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573899.202381:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573899.202386:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810185600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573899.202392:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573899.202394:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.202398:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810185600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573899.202416:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573899.202419:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573899.202422:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573899.202423:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573899.202424:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573899.202426:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573899.202427:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14369 00000400:00000200:0.0:1788573899.202430:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573899.202432:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573899.202434:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573899.202436:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202438:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202441:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202442:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573899.202444:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573899.202445:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573899.202446:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573899.202454:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573899.202457:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573899.202461:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573899.202466:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202471:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202475:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202479:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573899.202491:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573899.202494:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202499:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573899.202505:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573899.202514:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573899.202521:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202528:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573899.202534:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573899.202541:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202548:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573899.202555:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573899.202573:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573899.202577:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573899.202579:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202584:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573899.202588:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573899.202593:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573899.202600:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573899.202608:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573899.202618:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810185728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573899.202631:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573899.202675:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.202687:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810185728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573899.202728:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573899.202734:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573899.202740:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573899.202743:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573899.202746:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573899.202749:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573899.202755:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14369 00000400:00000200:0.0:1788573899.202763:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573899.202769:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573899.202772:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573899.202779:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202784:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202790:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202794:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573899.202798:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573899.202801:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573899.202803:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573899.202807:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573899.202811:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573899.202815:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573899.202820:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202823:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.202828:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202832:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573899.202836:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573899.202843:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573899.202847:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573899.202859:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573899.202866:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573899.202873:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202876:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573899.202882:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573899.202888:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202893:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573899.202899:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573899.202906:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573899.202913:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573899.202916:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.202919:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573899.202920:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573899.202922:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573899.202928:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573899.202935:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573899.202950:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810185856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573899.202981:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573899.202985:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.202992:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810185856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573899.203009:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573899.203014:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573899.203022:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573899.203024:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573899.203028:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573899.203033:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573899.203038:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14369 00000400:00000200:0.0:1788573899.203044:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573899.203049:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573899.203054:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573899.203066:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203070:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.203076:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203082:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573899.203084:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573899.203087:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573899.203092:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573899.203095:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573899.203099:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573899.203104:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573899.203109:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203110:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.203111:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203113:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573899.203119:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573899.203122:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203123:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573899.203124:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573899.203128:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573899.203132:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203137:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573899.203141:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573899.203146:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203149:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573899.203151:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573899.203153:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573899.203159:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573899.203161:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203163:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573899.203164:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573899.203166:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573899.203173:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788573899.203185:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573899.203194:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810185984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573899.203205:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573899.203209:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.203216:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810185984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573899.203229:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573899.203233:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573899.203241:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573899.203244:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573899.203248:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573899.203257:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573899.203265:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14369 00000400:00000200:0.0:1788573899.203277:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573899.203286:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573899.203290:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573899.203297:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203301:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.203302:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203304:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573899.203308:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573899.203312:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573899.203315:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573899.203318:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573899.203323:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573899.203325:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573899.203331:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203335:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573899.203344:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203349:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573899.203352:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573899.203359:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573899.203364:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573899.203367:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573899.203371:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573899.203375:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203379:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573899.203382:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573899.203388:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203391:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573899.203398:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573899.203404:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573899.203408:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573899.203411:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573899.203416:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573899.203419:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573899.203422:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573899.203429:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788573899.203435:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573899.203446:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810186112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573899.203464:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573899.203468:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.203475:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810186112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573899.203488:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.203494:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14375) 00000020:00000001:0.0:1788573902.017849:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573902.017854:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573902.017858:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573903.041753:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309042015, at 4309042048 00000100:00000010:0.0:1788573903.041770:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788573903.041815:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573903.041820:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573903.041824:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573904.705788:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573904.705798:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573904.705801:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573904.705805:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573904.705811:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573904.705820:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14370 00000400:00000200:0.0:1788573904.705829:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573904.705836:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573904.705840:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573904.705844:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.705847:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.705850:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.705853:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573904.705858:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573904.705861:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573904.705863:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573904.705867:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573904.705870:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573904.705872:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573904.705876:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.705878:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.705880:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.705883:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573904.705886:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573904.705890:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573904.705913:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573904.705917:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573904.705920:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573904.705924:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.705926:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573904.705929:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573904.705935:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573904.705939:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573904.705943:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573904.705946:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.705949:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573904.705953:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573904.705957:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573904.705965:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573904.705971:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.705981:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810186240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573904.705992:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573904.705996:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.706002:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810186240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.706023:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573904.706027:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573904.706030:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573904.706032:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573904.706034:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573904.706036:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573904.706040:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14370 00000400:00000200:0.0:1788573904.706045:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573904.706050:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573904.706053:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573904.706058:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706077:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706085:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706091:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573904.706096:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573904.706100:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573904.706102:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573904.706107:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573904.706113:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573904.706119:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573904.706129:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706133:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706139:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706145:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573904.706150:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573904.706155:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706160:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573904.706164:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573904.706173:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573904.706177:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706179:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573904.706185:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573904.706193:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706197:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573904.706202:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573904.706206:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573904.706210:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573904.706215:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706219:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573904.706223:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573904.706227:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573904.706233:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573904.706240:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.706256:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810186368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573904.706272:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573904.706276:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.706283:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810186368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.706299:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573904.706305:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573904.706312:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573904.706318:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573904.706323:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573904.706328:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573904.706331:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14370 00000400:00000200:0.0:1788573904.706374:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573904.706383:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573904.706396:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573904.706403:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706409:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706415:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706418:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573904.706423:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573904.706426:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573904.706429:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573904.706434:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573904.706439:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573904.706444:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573904.706481:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706486:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706490:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706500:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573904.706503:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573904.706507:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706512:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573904.706514:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573904.706518:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573904.706522:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706526:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573904.706529:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573904.706534:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706539:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573904.706545:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573904.706551:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573904.706558:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573904.706563:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706568:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573904.706570:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573904.706572:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573904.706578:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573904.706585:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.706596:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810186496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573904.706611:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573904.706615:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.706622:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810186496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.706657:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573904.706660:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573904.706665:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573904.706667:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573904.706669:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573904.706679:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573904.706686:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14370 00000400:00000200:0.0:1788573904.706696:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573904.706703:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573904.706708:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573904.706713:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706718:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706724:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706730:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573904.706734:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573904.706737:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573904.706740:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573904.706743:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573904.706748:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573904.706753:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573904.706761:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706766:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706771:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706776:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573904.706780:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573904.706786:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706791:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573904.706793:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573904.706797:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573904.706800:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706803:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573904.706808:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573904.706812:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706813:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573904.706818:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573904.706827:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573904.706830:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573904.706833:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.706835:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573904.706837:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573904.706839:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573904.706845:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573904.706850:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.706860:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810186624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573904.706868:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573904.706869:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.706874:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810186624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.706887:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573904.706891:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573904.706896:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573904.706900:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573904.706902:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573904.706908:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573904.706912:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14370 00000400:00000200:0.0:1788573904.706920:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573904.706922:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573904.706924:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573904.706926:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706929:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706933:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706936:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573904.706939:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573904.706941:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573904.706951:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573904.706955:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573904.706961:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573904.706968:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573904.706975:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706980:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573904.706987:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573904.706994:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573904.706999:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573904.707005:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573904.707009:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573904.707013:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573904.707016:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573904.707020:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.707022:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573904.707025:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573904.707029:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.707032:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573904.707037:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573904.707040:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573904.707045:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573904.707047:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.707049:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573904.707053:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573904.707056:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573904.707061:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573904.707066:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.707075:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810186752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573904.707081:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573904.707086:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.707094:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810186752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.707107:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.707112:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14381) 00000020:00000001:0.0:1788573907.137815:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573907.137819:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573907.137822:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573908.161814:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573908.161821:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573908.161826:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573908.545758:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309047518, at 4309047552 00000100:00000010:0.0:1788573908.545773:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00080000:0.0:1788573909.825712:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573909.825722:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573909.825724:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573909.825727:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573909.825731:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573909.825736:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14376 00000400:00000200:0.0:1788573909.825744:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573909.825749:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573909.825752:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573909.825754:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825757:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.825759:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825761:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573909.825764:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573909.825765:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573909.825767:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573909.825770:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573909.825772:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573909.825774:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573909.825776:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825793:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.825795:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825796:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573909.825799:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573909.825801:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825803:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573909.825805:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573909.825807:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573909.825810:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.825811:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573909.825813:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573909.825819:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573909.825821:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573909.825825:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573909.825826:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.825828:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573909.825831:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573909.825836:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573909.825845:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573909.825853:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.825862:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810186880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573909.825875:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573909.825879:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.825885:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810186880/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573909.825904:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573909.825907:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573909.825909:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573909.825910:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573909.825912:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573909.825913:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573909.825922:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14376 00000400:00000200:0.0:1788573909.825925:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573909.825928:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573909.825936:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573909.825939:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825941:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.825943:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825945:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573909.825947:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573909.825948:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573909.825949:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573909.825951:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573909.825956:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573909.825959:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573909.825964:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825969:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.825970:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825973:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573909.825976:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573909.825979:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573909.825984:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573909.825988:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573909.825997:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573909.826001:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826004:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573909.826008:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573909.826011:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826017:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573909.826024:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573909.826028:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573909.826033:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573909.826039:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826042:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573909.826043:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573909.826048:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573909.826052:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573909.826056:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.826062:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810187008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573909.826072:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573909.826074:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.826077:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810187008/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573909.826087:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573909.826098:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573909.826104:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573909.826106:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573909.826107:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573909.826109:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573909.826112:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14376 00000400:00000200:0.0:1788573909.826118:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573909.826120:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573909.826122:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573909.826128:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826132:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826135:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826138:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573909.826140:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573909.826142:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573909.826144:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573909.826146:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573909.826152:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573909.826163:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573909.826173:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826179:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826184:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826188:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573909.826194:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573909.826200:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826203:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573909.826207:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573909.826212:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573909.826215:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826218:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573909.826220:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573909.826225:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826226:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573909.826230:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573909.826234:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573909.826237:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573909.826240:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826241:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573909.826244:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573909.826247:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573909.826251:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573909.826256:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.826264:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810187136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573909.826276:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573909.826279:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.826283:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810187136/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573909.826299:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573909.826308:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573909.826314:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573909.826315:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573909.826317:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573909.826319:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573909.826324:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14376 00000400:00000200:0.0:1788573909.826330:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573909.826335:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573909.826338:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573909.826342:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826345:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826349:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826351:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573909.826355:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573909.826357:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573909.826359:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573909.826363:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573909.826367:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573909.826385:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573909.826395:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826401:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826407:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826413:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573909.826418:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573909.826423:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826429:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573909.826435:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573909.826444:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573909.826449:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826454:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573909.826464:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573909.826469:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826474:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573909.826479:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573909.826484:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573909.826487:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573909.826490:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826494:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573909.826495:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573909.826497:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573909.826502:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573909.826509:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.826517:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810187264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573909.826527:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573909.826529:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.826533:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810187264/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573909.826540:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573909.826546:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573909.826549:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573909.826551:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573909.826553:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573909.826558:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573909.826560:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14376 00000400:00000200:0.0:1788573909.826567:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573909.826572:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573909.826575:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573909.826578:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826583:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826587:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826595:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573909.826599:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573909.826601:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573909.826604:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573909.826606:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573909.826610:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573909.826615:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573909.826621:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826624:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573909.826626:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826631:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573909.826652:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573909.826654:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573909.826656:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573909.826657:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573909.826659:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573909.826660:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826662:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573909.826664:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573909.826666:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826667:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573909.826670:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573909.826672:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573909.826676:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573909.826679:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.826681:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573909.826685:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573909.826687:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573909.826693:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573909.826698:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.826706:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810187392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573909.826739:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573909.826741:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.826746:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810187392/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573909.826766:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.826771:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14386) 00000800:00000001:0.0:1788573911.297931:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573911.297951:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573911.297957:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573911.297960:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573912.257890:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573912.257895:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573912.257899:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573913.281878:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573913.281884:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573913.281890:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573913.665734:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309052638, at 4309052672 00000100:00000010:0.0:1788573913.665748:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788573917.377816:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573917.377820:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573917.377823:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573918.401906:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573918.401911:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573918.401914:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573918.785783:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309057758, at 4309057792 00000100:00000010:0.0:1788573918.785802:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788573922.497895:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573922.497902:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573922.497908:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573923.521884:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573923.521891:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573923.521896:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573924.737793:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309063711, at 4309063744 00000100:00000010:0.0:1788573924.737813:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788573927.617806:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573927.617811:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573927.617814:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573928.641847:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573928.641850:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573928.641853:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573930.049765:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309069022, at 4309069056 00000100:00000010:0.0:1788573930.049787:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788573932.737835:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573932.737839:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573932.737843:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573933.761851:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573933.761854:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573933.761857:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573935.169720:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309074142, at 4309074176 00000100:00000010:0.0:1788573935.169729:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788573936.449792:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573936.449800:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573936.449802:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573936.449806:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573936.449809:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573936.449815:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14403 00000400:00000200:0.0:1788573936.449822:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573936.449827:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573936.449830:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573936.449833:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449835:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.449837:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449839:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573936.449842:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573936.449857:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573936.449860:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573936.449862:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573936.449864:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573936.449866:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573936.449868:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449870:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.449871:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449872:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573936.449875:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573936.449878:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449880:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573936.449882:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573936.449885:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573936.449887:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.449889:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573936.449890:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573936.449896:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788573936.449899:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573936.449902:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573936.449903:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.449906:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573936.449908:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573936.449912:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573936.449918:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788573936.449924:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.449931:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810190848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573936.449939:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573936.449942:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.449946:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810190848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573936.449971:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573936.449974:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573936.449976:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573936.449977:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573936.449978:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573936.449980:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573936.449982:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14403 00000400:00000200:0.0:1788573936.449984:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573936.449986:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573936.449988:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573936.449990:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449991:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.449992:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.449994:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573936.449996:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573936.449997:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573936.449998:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573936.450000:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573936.450001:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573936.450003:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573936.450006:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450007:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450009:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450010:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573936.450012:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573936.450014:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450016:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573936.450023:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573936.450029:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573936.450032:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450033:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573936.450038:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573936.450044:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450048:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573936.450053:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788573936.450056:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573936.450060:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573936.450063:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450065:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573936.450067:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573936.450069:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573936.450073:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788573936.450077:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.450085:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810190976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573936.450094:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573936.450098:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.450104:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810190976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573936.450113:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573936.450118:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573936.450122:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573936.450123:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573936.450125:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573936.450127:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573936.450133:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14403 00000400:00000200:0.0:1788573936.450140:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573936.450147:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573936.450151:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573936.450164:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450167:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450170:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450172:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573936.450177:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573936.450180:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573936.450183:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573936.450185:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573936.450189:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573936.450195:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573936.450203:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450208:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450212:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450218:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573936.450223:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573936.450229:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450235:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573936.450239:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573936.450245:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573936.450250:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450254:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573936.450258:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573936.450264:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450267:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573936.450272:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788573936.450277:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573936.450282:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573936.450286:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450288:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573936.450291:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573936.450295:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573936.450301:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788573936.450317:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.450327:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810191104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573936.450345:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573936.450349:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.450353:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810191104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573936.450369:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573936.450373:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573936.450379:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573936.450381:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573936.450383:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573936.450386:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573936.450391:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14403 00000400:00000200:0.0:1788573936.450401:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573936.450408:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573936.450413:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573936.450419:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450424:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450431:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450436:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573936.450440:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573936.450443:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573936.450446:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573936.450448:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573936.450452:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573936.450457:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573936.450465:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450469:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450479:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450482:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573936.450487:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573936.450494:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450501:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573936.450505:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573936.450510:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573936.450513:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450515:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573936.450518:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573936.450522:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450526:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573936.450530:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788573936.450537:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573936.450544:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573936.450547:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450551:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573936.450555:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573936.450558:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573936.450562:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788573936.450569:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.450581:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810191232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573936.450595:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573936.450600:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.450606:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810191232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573936.450620:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573936.450626:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573936.450675:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573936.450694:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573936.450704:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573936.450711:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573936.450718:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14403 00000400:00000200:0.0:1788573936.450727:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573936.450733:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573936.450737:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573936.450744:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450749:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450753:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450757:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573936.450761:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573936.450764:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573936.450768:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573936.450772:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573936.450777:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573936.450782:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573936.450791:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450794:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573936.450799:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450804:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573936.450809:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573936.450816:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573936.450819:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573936.450821:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573936.450826:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573936.450831:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450835:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573936.450839:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573936.450843:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450849:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573936.450861:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788573936.450864:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573936.450867:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573936.450869:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573936.450871:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573936.450873:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573936.450875:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573936.450879:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788573936.450883:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.450891:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810191360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573936.450907:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573936.450913:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.450922:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810191360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573936.450939:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573936.450947:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14413) 00000020:00000001:0.0:1788573937.857830:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573937.857833:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573937.857836:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573938.881864:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573938.881868:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573938.881871:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573940.289736:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309079262, at 4309079296 00000100:00000010:0.0:1788573940.289752:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788573942.977810:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573942.977815:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573942.977819:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573944.001896:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573944.001901:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573944.001906:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573946.753773:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309085727, at 4309085760 00000100:00000010:0.0:1788573946.753795:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788573948.097831:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573948.097835:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573948.097837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573948.225710:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573948.225718:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573948.225720:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573948.225723:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573948.225727:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573948.225734:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14414 00000400:00000200:0.0:1788573948.225741:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573948.225746:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573948.225749:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573948.225751:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225753:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.225755:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225757:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573948.225760:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573948.225761:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573948.225764:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573948.225766:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573948.225768:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573948.225769:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573948.225771:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225773:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.225774:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225775:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573948.225778:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573948.225780:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225782:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573948.225784:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573948.225800:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573948.225803:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.225805:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573948.225807:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573948.225812:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788573948.225844:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573948.225847:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573948.225849:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.225851:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573948.225854:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573948.225858:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573948.225865:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788573948.225870:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.225878:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810192896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573948.225886:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573948.225889:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.225893:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810192896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573948.225917:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573948.225920:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573948.225923:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573948.225925:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573948.225926:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573948.225929:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573948.225931:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14414 00000400:00000200:0.0:1788573948.225935:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573948.225938:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573948.225940:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573948.225943:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225945:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.225953:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225956:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573948.225959:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573948.225961:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573948.225963:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573948.225966:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573948.225969:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573948.225973:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573948.225978:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225981:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.225985:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.225987:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573948.225992:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573948.225997:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226002:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573948.226005:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573948.226011:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573948.226016:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226019:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573948.226025:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573948.226030:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226035:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573948.226039:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788573948.226044:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573948.226049:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573948.226050:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226051:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573948.226053:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573948.226057:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573948.226063:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573948.226069:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.226078:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810193024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573948.226096:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573948.226100:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.226106:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810193024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573948.226124:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573948.226127:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573948.226129:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573948.226132:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573948.226134:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573948.226140:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573948.226144:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14414 00000400:00000200:0.0:1788573948.226152:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573948.226158:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573948.226159:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573948.226163:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226169:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226172:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226176:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573948.226180:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573948.226182:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573948.226184:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573948.226187:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573948.226189:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573948.226193:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573948.226197:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226200:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226205:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226210:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573948.226219:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573948.226228:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226231:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573948.226234:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573948.226237:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573948.226243:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226246:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573948.226249:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573948.226252:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226256:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573948.226263:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788573948.226268:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573948.226271:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573948.226273:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226277:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573948.226278:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573948.226280:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573948.226283:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573948.226286:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.226293:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810193152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573948.226304:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573948.226306:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.226311:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810193152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573948.226321:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573948.226323:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573948.226325:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573948.226326:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573948.226327:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573948.226329:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573948.226337:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14414 00000400:00000200:0.0:1788573948.226341:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573948.226344:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573948.226347:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573948.226350:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226354:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226356:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226358:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573948.226362:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573948.226363:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573948.226365:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573948.226367:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573948.226372:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573948.226376:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573948.226382:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226385:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226388:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226392:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573948.226406:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573948.226409:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226412:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573948.226414:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573948.226417:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573948.226419:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226423:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573948.226425:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573948.226428:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226431:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573948.226436:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573948.226440:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573948.226442:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573948.226452:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226456:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573948.226460:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573948.226465:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573948.226475:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573948.226482:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.226493:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810193280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573948.226512:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573948.226516:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.226524:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810193280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573948.226540:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573948.226543:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573948.226552:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573948.226553:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573948.226557:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573948.226561:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573948.226568:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14414 00000400:00000200:0.0:1788573948.226576:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573948.226583:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573948.226588:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573948.226593:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226597:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226600:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226604:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573948.226607:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573948.226610:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573948.226611:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573948.226613:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573948.226615:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573948.226624:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573948.226631:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226669:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573948.226674:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226678:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573948.226682:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573948.226687:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573948.226690:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573948.226697:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573948.226702:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573948.226706:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226708:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573948.226712:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573948.226714:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226720:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573948.226723:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573948.226727:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573948.226730:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573948.226732:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573948.226736:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573948.226738:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573948.226739:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573948.226742:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573948.226745:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.226751:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810193408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573948.226769:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573948.226773:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.226779:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810193408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573948.226791:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573948.226802:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14424) 00000020:00000001:0.0:1788573949.121810:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573949.121814:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573949.121817:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573952.065766:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309091038, at 4309091072 00000100:00000010:0.0:1788573952.065787:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788573953.217856:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573953.217860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573953.217864:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573953.345737:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573953.345748:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573953.345751:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573953.345755:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573953.345759:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573953.345767:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14419 00000400:00000200:0.0:1788573953.345776:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573953.345784:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573953.345787:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573953.345790:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345793:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.345795:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345797:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573953.345801:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573953.345803:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788573953.345806:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573953.345809:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788573953.345811:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788573953.345813:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788573953.345815:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345817:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.345818:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345841:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573953.345844:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788573953.345847:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345849:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573953.345852:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573953.345855:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573953.345857:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.345859:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573953.345861:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573953.345867:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788573953.345871:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788573953.345890:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573953.345891:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.345894:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573953.345897:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573953.345902:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573953.345911:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788573953.345918:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.345928:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810193536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573953.345937:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573953.345942:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.345946:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810193536/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573953.345967:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573953.345970:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573953.345972:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573953.345974:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573953.345975:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573953.345977:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573953.345979:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14419 00000400:00000200:0.0:1788573953.345988:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573953.345992:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573953.345994:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573953.345997:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.345999:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346001:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346003:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573953.346006:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573953.346007:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788573953.346009:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573953.346011:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788573953.346014:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788573953.346017:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788573953.346021:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346025:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346029:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346031:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573953.346033:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788573953.346039:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346045:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573953.346049:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573953.346056:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573953.346059:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346063:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573953.346070:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573953.346073:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346075:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573953.346077:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788573953.346081:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573953.346084:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573953.346092:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346094:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573953.346098:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573953.346102:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573953.346109:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788573953.346115:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.346123:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810193664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573953.346133:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573953.346136:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.346139:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810193664/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573953.346149:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573953.346155:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573953.346160:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573953.346162:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573953.346166:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573953.346170:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573953.346175:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14419 00000400:00000200:0.0:1788573953.346183:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573953.346191:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573953.346194:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573953.346200:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346202:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346207:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346212:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573953.346214:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573953.346215:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788573953.346217:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573953.346218:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788573953.346222:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788573953.346232:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788573953.346238:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346243:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346249:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346252:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573953.346256:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788573953.346262:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346267:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573953.346273:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573953.346277:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573953.346280:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346282:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573953.346284:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573953.346289:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346292:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573953.346297:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788573953.346301:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788573953.346305:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573953.346306:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346308:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573953.346310:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573953.346311:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573953.346315:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788573953.346318:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.346324:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810193792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573953.346338:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573953.346343:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.346348:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810193792/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573953.346353:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573953.346361:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573953.346365:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573953.346369:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573953.346371:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573953.346377:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573953.346382:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14419 00000400:00000200:0.0:1788573953.346389:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573953.346393:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573953.346394:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573953.346399:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346402:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346405:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346409:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573953.346414:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573953.346417:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788573953.346421:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573953.346425:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788573953.346428:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788573953.346441:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788573953.346444:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346448:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346452:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346455:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573953.346459:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788573953.346465:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346471:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573953.346475:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573953.346481:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573953.346486:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346492:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573953.346503:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573953.346511:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346516:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573953.346526:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788573953.346531:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788573953.346536:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573953.346550:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346553:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573953.346557:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573953.346560:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573953.346568:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788573953.346575:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.346584:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810193920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573953.346601:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573953.346607:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.346618:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810193920/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573953.346664:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573953.346672:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573953.346680:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573953.346686:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573953.346691:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573953.346698:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573953.346706:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14419 00000400:00000200:0.0:1788573953.346718:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788573953.346726:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573953.346730:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573953.346737:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346743:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346749:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346764:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788573953.346769:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573953.346770:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788573953.346772:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573953.346775:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788573953.346777:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788573953.346784:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788573953.346792:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346797:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573953.346803:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346809:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788573953.346815:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788573953.346824:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788573953.346829:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573953.346833:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573953.346839:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573953.346844:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346850:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573953.346854:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573953.346860:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346865:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573953.346887:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788573953.346893:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788573953.346900:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573953.346904:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.346909:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573953.346913:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573953.346917:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573953.346926:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788573953.346934:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.346944:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810194048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573953.346981:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573953.346986:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.346996:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810194048/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573953.347020:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573953.347029:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14430) 00000020:00000001:0.0:1788573954.241826:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573954.241832:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573954.241836:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573957.185708:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309096158, at 4309096192 00000100:00000010:0.0:1788573957.185727:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788573958.337899:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573958.337906:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573958.337912:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573959.361973:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573959.361979:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573959.361982:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573962.305743:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309101278, at 4309101312 00000100:00000010:0.0:1788573962.305763:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788573963.457785:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573963.457788:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573963.457791:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573964.481752:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573964.481758:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573964.481766:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573964.481771:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573964.481775:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573964.481780:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573964.481784:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573964.481788:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573964.481792:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573964.481798:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573964.481802:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573964.481824:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573964.481828:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573964.481833:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573964.481835:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573964.481840:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573964.481846:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788573964.481853:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573964.481857:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573964.481861:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573964.481868:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692500. 00010000:00000001:0.0:1788573964.481873:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573964.481876:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788573964.481879:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573964.481882:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168b84 to hash 00000020:00000001:0.0:1788573964.481883:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573964.481884:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573964.481887:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573964.481890:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573964.481892:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573964.481895:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573964.481903:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573964.481912:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573964.481916:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573964.481918:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573964.481923:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573964.481925:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788573964.481927:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573964.481928:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788573964.481935:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573964.481939:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573964.481941:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573964.481942:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573964.481951:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573964.481958:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573964.481962:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573964.481966:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573964.481969:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573964.481974:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573964.481976:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573964.481977:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573964.481979:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788573964.481980:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573964.481982:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573964.481985:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573964.481986:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573964.481988:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573964.481990:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573964.481996:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810195456:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573964.481999:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573964.482015:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573964.482017:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573964.482019:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573964.482022:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573964.482036:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573964.482038:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573964.482041:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573964.482042:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573964.482045:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573964.482047:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573964.482049:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573964.482050:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788573964.482052:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573964.482054:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788573964.482057:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482062:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573964.482065:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482069:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573964.482072:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482075:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573964.482077:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573964.482079:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482082:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573964.482083:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573964.482085:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573964.482087:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573964.482098:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573964.482100:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573964.482102:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482105:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573964.482106:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573964.482107:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573964.482108:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788573964.482109:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573964.482110:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788573964.482112:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573964.482114:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573964.482117:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573964.482119:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573964.482122:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573964.482124:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573964.482125:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573964.482127:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573964.482128:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573964.482131:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573964.482132:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573964.482133:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573964.482134:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573964.482136:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168b84 from hash 00010000:00000001:0.0:1788573964.482137:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573964.482138:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573964.482139:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573964.482142:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573964.482143:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573964.482145:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573964.482146:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573964.482147:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573964.482149:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573964.482151:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573964.482154:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573964.482161:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573964.482162:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573964.482163:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573964.482165:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573964.482166:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573964.482168:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573964.482169:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573964.482171:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168b84 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573964.482179:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692500 count: 0 00010000:00000001:0.0:1788573964.482187:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573964.482189:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573964.482192:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573964.482197:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810195456/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573964.482205:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573964.482207:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573964.482208:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573964.482210:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573964.482213:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788573964.482215:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573964.482217:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573964.482219:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788573964.482221:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573964.482222:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573964.482228:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573964.482229:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573964.482232:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788573968.321784:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309107295, at 4309107328 00000100:00000010:0.0:1788573968.321803:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788573968.577835:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573968.577842:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573968.577847:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573973.569808:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309112542, at 4309112576 00000100:00000010:0.0:1788573973.569827:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788573973.697830:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573973.697834:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573973.697837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573978.689788:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309117662, at 4309117696 00000100:00000010:0.0:1788573978.689809:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788573978.817904:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573978.817909:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573978.817913:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573983.809760:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309122782, at 4309122816 00000100:00000010:0.0:1788573983.809778:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788573983.937861:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573983.937867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573983.937870:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788573984.577774:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788573984.577790:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9440. 00000020:00000010:0.0:1788573984.577798:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788573984.577803:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788573984.577807:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788573984.577813:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573984.577820:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788573984.577825:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573984.577830:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788573984.577834:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788573984.577852:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788573984.577855:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788573984.577860:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788573984.577867:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788573984.577870:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788573984.577874:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788573984.577877:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788573984.577880:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788573984.577883:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788573984.577886:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788573984.577889:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9440. 10000000:01000000:0.0:1788573984.577892:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788573984.577895:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788573984.577898:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788573984.577900:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788573984.577902:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788573984.577906:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788573984.577908:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573984.577911:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788573984.577913:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573984.577916:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788573984.577919:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788573984.577920:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573984.577923:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788573985.601758:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788573985.601764:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788573985.601769:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573985.601772:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788573985.601774:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788573985.601775:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788573985.601778:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788573985.601780:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788573985.601783:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788573985.601786:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788573985.601787:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788573985.601798:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788573985.601824:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788573985.601830:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573985.601834:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573985.601839:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573985.601846:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788573985.601854:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573985.601859:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788573985.601862:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788573985.601870:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692800. 00010000:00000001:0.0:1788573985.601874:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788573985.601877:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788573985.601882:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788573985.601885:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168b8b to hash 00000020:00000001:0.0:1788573985.601887:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788573985.601889:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573985.601892:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573985.601895:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573985.601898:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573985.601903:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573985.601913:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573985.601924:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788573985.601930:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788573985.601932:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788573985.601938:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573985.601940:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788573985.601943:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573985.601945:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788573985.601949:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573985.601958:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573985.601961:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788573985.601963:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573985.601973:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573985.601982:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788573985.601986:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573985.601993:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573985.601997:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573985.602003:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573985.602006:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573985.602008:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573985.602010:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788573985.602013:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573985.602015:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788573985.602019:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573985.602021:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573985.602024:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573985.602027:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573985.602035:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810198784:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788573985.602041:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573985.602044:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573985.602047:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788573985.602049:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573985.602052:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573985.602059:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573985.602066:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788573985.602072:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788573985.602073:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788573985.602076:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788573985.602080:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573985.602082:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573985.602084:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788573985.602087:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573985.602090:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788573985.602094:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602101:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788573985.602105:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602112:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788573985.602115:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602123:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573985.602125:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788573985.602128:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602134:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573985.602136:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573985.602140:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788573985.602164:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788573985.602172:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788573985.602181:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573985.602184:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602189:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788573985.602190:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788573985.602191:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573985.602193:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788573985.602194:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573985.602195:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788573985.602197:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573985.602200:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788573985.602204:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788573985.602206:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788573985.602210:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788573985.602211:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788573985.602212:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573985.602214:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788573985.602216:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573985.602218:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573985.602219:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573985.602220:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573985.602221:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573985.602223:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168b8b from hash 00010000:00000001:0.0:1788573985.602224:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573985.602225:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573985.602226:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573985.602227:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573985.602228:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573985.602229:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788573985.602231:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573985.602235:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573985.602236:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788573985.602237:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573985.602240:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788573985.602243:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573985.602244:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573985.602245:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788573985.602246:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788573985.602247:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573985.602248:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573985.602248:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573985.602250:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168b8b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788573985.602254:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692800 count: 0 00010000:00000001:0.0:1788573985.602262:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573985.602263:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788573985.602265:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573985.602268:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810198784/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573985.602273:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788573985.602276:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573985.602277:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788573985.602278:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573985.602280:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788573985.602283:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573985.602284:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573985.602286:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788573985.602289:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573985.602290:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788573985.602291:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788573985.602293:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788573985.602296:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788573989.057886:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573989.057906:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573989.057910:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573989.121960:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573989.121968:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573989.121981:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573989.121987:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573989.121989:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573989.441728:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309128415, at 4309128448 00000100:00000010:0.0:1788573989.441742:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788573994.177876:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573994.177880:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573994.177884:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573994.561779:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309133534, at 4309133568 00000100:00000010:0.0:1788573994.561796:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788573999.297950:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573999.297954:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573999.297957:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788573999.681767:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309138654, at 4309138688 00000100:00000010:0.0:1788573999.681787:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788574004.417854:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574004.417858:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574004.417861:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574004.801732:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309143775, at 4309143808 00000100:00000010:0.0:1788574004.801752:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574009.537883:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574009.537886:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574009.537889:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574011.841719:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309150815, at 4309150848 00000100:00000010:0.0:1788574011.841732:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000020:00000001:0.0:1788574014.657875:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574014.657879:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574014.657882:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574017.089771:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309156063, at 4309156096 00000100:00000010:0.0:1788574017.089807:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574019.777847:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574019.777851:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574019.777853:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788574021.441915:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788574021.441953:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574021.441961:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574021.441964:0:2391:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788574021.441967:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574021.441971:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574021.441974:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574021.441977:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788574021.441982:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574021.441985:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574021.441991:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3220 00000400:00000010:0.0:1788574021.441993:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3220. 00000100:00000001:0.0:1788574021.441997:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574021.442004:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6a680 x1875440810203520/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442029:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574021.442034:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a680 x1875440810203520/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442048:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.442050:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574021.442059:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574021.442062:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788574021.442065:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574021.442068:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574021.442072:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3000 00000400:00000010:0.0:1788574021.442074:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3000. 00000100:00000001:0.0:1788574021.442076:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574021.442081:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6b100 x1875440810203648/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442104:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574021.442109:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b100 x1875440810203648/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442116:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.442119:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574021.442128:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574021.442134:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9100. 00000400:00000200:0.0:1788574021.442139:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574021.442146:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574021.442152:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3aa0 00000400:00000010:0.0:1788574021.442156:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3aa0. 00000100:00000001:0.0:1788574021.442163:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574021.442181:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6ad80 x1875440810203776/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442198:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574021.442206:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6ad80 x1875440810203776/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442218:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.442223:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574021.442234:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574021.442241:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9e00. 00000400:00000200:0.0:1788574021.442246:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574021.442255:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574021.442265:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3ee0 00000400:00000010:0.0:1788574021.442269:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3ee0. 00000100:00000001:0.0:1788574021.442272:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574021.442291:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68700 x1875440810203904/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442309:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574021.442318:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68700 x1875440810203904/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442337:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.442341:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574021.442354:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574021.442361:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9300. 00000400:00000200:0.0:1788574021.442367:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574021.442371:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574021.442378:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3b28 00000400:00000010:0.0:1788574021.442383:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3b28. 00000100:00000001:0.0:1788574021.442389:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574021.442398:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69180 x1875440810204032/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442416:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574021.442432:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69180 x1875440810204032/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574021.442449:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.442454:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788574022.209739:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309161182, at 4309161216 00000100:00000010:0.0:1788574022.209757:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00080000:0.0:1788574023.489763:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574023.489771:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574023.489773:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574023.489775:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574023.489779:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574023.489784:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14490 00000400:00000200:0.0:1788574023.489791:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574023.489796:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574023.489798:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574023.489801:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489803:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.489817:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489818:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574023.489822:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574023.489823:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574023.489825:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574023.489828:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574023.489829:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574023.489831:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574023.489833:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489835:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.489836:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489837:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574023.489840:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574023.489842:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489844:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574023.489846:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574023.489849:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574023.489851:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.489852:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574023.489854:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574023.489859:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574023.489861:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574023.489864:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574023.489865:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.489868:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574023.489870:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574023.489874:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574023.489880:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574023.489886:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.489894:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810204160/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574023.489904:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574023.489919:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.489926:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810204160/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574023.489948:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574023.489952:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574023.489955:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574023.489957:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574023.489959:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574023.489962:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574023.489965:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14490 00000400:00000200:0.0:1788574023.489969:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574023.489973:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574023.489976:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574023.489980:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489983:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.489986:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.489988:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574023.489991:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574023.489993:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574023.489995:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574023.490001:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574023.490004:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574023.490007:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574023.490011:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490016:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490020:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490023:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574023.490028:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574023.490035:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490052:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574023.490056:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574023.490065:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574023.490073:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490078:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574023.490087:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574023.490092:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490095:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574023.490101:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574023.490106:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574023.490111:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574023.490112:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490115:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574023.490127:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574023.490132:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574023.490140:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574023.490146:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.490156:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810204288/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574023.490164:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574023.490166:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.490171:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810204288/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574023.490204:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574023.490211:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574023.490219:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574023.490224:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574023.490228:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574023.490235:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574023.490241:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14490 00000400:00000200:0.0:1788574023.490251:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574023.490264:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574023.490270:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574023.490276:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490279:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490297:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490301:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574023.490305:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574023.490308:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574023.490310:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574023.490314:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574023.490319:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574023.490324:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574023.490332:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490336:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490338:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490340:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574023.490346:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574023.490353:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490359:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574023.490364:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574023.490370:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574023.490374:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490376:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574023.490378:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574023.490382:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490386:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574023.490392:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574023.490397:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574023.490403:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574023.490406:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490410:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574023.490421:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574023.490425:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574023.490434:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574023.490441:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.490452:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810204416/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574023.490468:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574023.490474:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.490484:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810204416/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574023.490497:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574023.490501:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574023.490507:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574023.490511:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574023.490513:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574023.490517:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574023.490523:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14490 00000400:00000200:0.0:1788574023.490530:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574023.490537:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574023.490540:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574023.490548:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490554:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490560:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490564:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574023.490569:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574023.490572:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574023.490576:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574023.490580:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574023.490586:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574023.490591:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574023.490605:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490610:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490615:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490620:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574023.490625:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574023.490631:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490670:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574023.490674:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574023.490679:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574023.490684:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490690:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574023.490693:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574023.490697:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490703:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574023.490710:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574023.490716:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574023.490722:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574023.490726:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490729:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574023.490733:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574023.490736:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574023.490744:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574023.490751:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.490761:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810204544/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574023.490777:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574023.490782:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.490789:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810204544/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574023.490807:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574023.490812:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574023.490825:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574023.490829:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574023.490832:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574023.490839:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574023.490846:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14490 00000400:00000200:0.0:1788574023.490855:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574023.490862:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574023.490868:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574023.490877:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490882:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490888:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490893:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574023.490899:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574023.490902:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574023.490907:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574023.490910:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574023.490915:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574023.490920:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574023.490927:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490934:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574023.490940:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490945:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574023.490951:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574023.490958:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574023.490965:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574023.490968:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574023.490973:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574023.490978:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490980:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574023.490982:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574023.490992:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.490996:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574023.491004:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574023.491009:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574023.491015:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574023.491018:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574023.491023:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574023.491026:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574023.491030:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574023.491040:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574023.491047:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.491059:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810204672/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574023.491072:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574023.491077:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.491082:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810204672/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574023.491095:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574023.491103:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14500) 00000020:00000001:0.0:1788574024.897816:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574024.897822:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574024.897825:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574027.329741:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309166302, at 4309166336 00000100:00000010:0.0:1788574027.329760:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788574030.017855:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574030.017860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574030.017864:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574033.473744:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309172447, at 4309172480 00000100:00000010:0.0:1788574033.473759:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788574035.137902:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574035.137912:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574035.137916:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574038.593766:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309177566, at 4309177600 00000100:00000010:0.0:1788574038.593785:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788574040.257787:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574040.257791:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574040.257796:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574043.713759:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309182687, at 4309182720 00000100:00000010:0.0:1788574043.713774:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788574045.377819:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574045.377822:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574045.377824:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574048.833772:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309187806, at 4309187840 00000100:00000010:0.0:1788574048.833794:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788574050.497882:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574050.497891:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574050.497896:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574055.233744:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309194207, at 4309194240 00000100:00000010:0.0:1788574055.233764:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788574055.617881:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574055.617886:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574055.617889:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574060.609734:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309199583, at 4309199616 00000100:00000010:0.0:1788574060.609750:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788574060.737810:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574060.737814:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574060.737817:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574065.729776:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309204702, at 4309204736 00000100:00000010:0.0:1788574065.729790:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788574065.857832:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574065.857837:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574065.857841:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574065.922079:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574070.849755:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309209822, at 4309209856 00000100:00000010:0.0:1788574070.849802:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574070.977806:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574070.977811:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574070.977813:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788574071.617760:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574071.617772:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9600. 00000020:00000010:0.0:1788574071.617814:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574071.617820:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574071.617823:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574071.617826:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574071.617831:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574071.617835:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574071.617839:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574071.617843:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788574071.617847:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574071.617849:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574071.617854:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574071.617859:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574071.617861:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574071.617864:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574071.617866:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574071.617869:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574071.617870:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574071.617872:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574071.617874:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9600. 10000000:01000000:0.0:1788574071.617876:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574071.617879:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574071.617881:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574071.617882:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574071.617883:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574071.617886:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574071.617887:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574071.617889:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574071.617903:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574071.617905:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574071.617907:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574071.617909:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574071.617910:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574073.601881:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574073.601897:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574073.601901:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574073.601903:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574073.601904:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574073.601905:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574073.601907:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574073.601909:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574073.601911:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574073.601914:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574073.601915:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574073.601925:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574073.601929:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574073.601934:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574073.601937:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574073.601942:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574073.601949:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574073.601954:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574073.601958:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574073.601960:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574073.601974:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692300. 00010000:00000001:0.0:1788574073.601977:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574073.601980:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788574073.601983:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574073.601986:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168ba7 to hash 00000020:00000001:0.0:1788574073.601988:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574073.601989:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788574073.601992:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788574073.601995:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574073.601997:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574073.602000:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574073.602020:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574073.602030:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574073.602034:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574073.602036:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574073.602041:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574073.602043:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574073.602046:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574073.602047:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574073.602051:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574073.602054:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574073.602056:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574073.602057:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574073.602095:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574073.602103:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574073.602107:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574073.602112:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574073.602115:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574073.602119:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574073.602121:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574073.602123:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574073.602124:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574073.602127:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574073.602128:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574073.602135:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574073.602137:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574073.602139:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574073.602142:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574073.602148:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810212096:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574073.602153:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574073.602155:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574073.602156:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574073.602158:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574073.602160:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574073.602165:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574073.602167:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574073.602172:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574073.602173:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574073.602175:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574073.602178:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574073.602179:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574073.602181:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788574073.602183:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788574073.602185:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788574073.602188:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602192:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574073.602195:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602199:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574073.602205:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602210:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574073.602211:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574073.602213:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602216:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574073.602218:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574073.602220:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574073.602222:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574073.602226:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574073.602228:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574073.602230:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602233:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574073.602234:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574073.602235:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574073.602236:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788574073.602237:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788574073.602239:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788574073.602241:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574073.602243:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574073.602247:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574073.602249:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574073.602253:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574073.602258:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574073.602259:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574073.602262:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574073.602263:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574073.602265:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574073.602267:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574073.602268:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574073.602269:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574073.602271:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168ba7 from hash 00010000:00000001:0.0:1788574073.602273:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574073.602274:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574073.602275:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574073.602276:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574073.602277:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574073.602278:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574073.602280:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574073.602280:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574073.602281:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574073.602283:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574073.602285:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574073.602288:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574073.602289:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574073.602290:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574073.602291:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574073.602292:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574073.602292:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574073.602293:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574073.602295:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168ba7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574073.602299:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692300 count: 0 00010000:00000001:0.0:1788574073.602307:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574073.602308:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574073.602309:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574073.602315:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810212096/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574073.602325:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574073.602327:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574073.602328:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574073.602331:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574073.602333:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574073.602336:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574073.602338:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574073.602339:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574073.602342:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574073.602343:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574073.602345:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574073.602347:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574073.602349:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574076.097879:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574076.097884:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574076.097888:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574077.441755:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309216415, at 4309216448 00000100:00000010:0.0:1788574077.441774:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788574081.217845:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574081.217852:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574081.217857:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574082.625746:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309221599, at 4309221632 00000100:00000010:0.0:1788574082.625771:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788574083.905822:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574083.905835:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574083.905838:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574083.905842:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574083.905847:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574083.905855:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14550 00000400:00000200:0.0:1788574083.905865:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574083.905873:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574083.905903:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574083.905906:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.905909:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.905911:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.905912:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574083.905916:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574083.905918:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574083.905921:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574083.905924:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574083.905926:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574083.905928:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574083.905930:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.905931:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.905933:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.905934:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574083.905936:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574083.905939:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574083.905941:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574083.905944:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574083.905947:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574083.905949:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.905951:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574083.905953:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574083.905960:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574083.905963:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574083.905966:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574083.905968:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.905970:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574083.905973:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574083.905978:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574083.906000:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574083.906007:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574083.906021:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810213504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574083.906047:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574083.906052:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.906056:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810213504/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574083.906081:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574083.906085:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574083.906089:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574083.906091:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574083.906092:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574083.906095:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574083.906099:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14550 00000400:00000200:0.0:1788574083.906105:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574083.906109:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574083.906113:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574083.906117:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906120:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906124:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906126:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574083.906130:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574083.906133:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574083.906135:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574083.906137:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574083.906139:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574083.906141:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574083.906149:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906154:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906157:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906167:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574083.906173:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574083.906181:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906188:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574083.906191:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574083.906197:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574083.906202:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906204:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574083.906208:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574083.906215:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906216:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574083.906220:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574083.906224:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574083.906226:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574083.906229:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906231:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574083.906232:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574083.906235:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574083.906243:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574083.906247:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574083.906255:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810213632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574083.906274:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574083.906278:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.906283:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810213632/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574083.906295:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574083.906300:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574083.906308:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574083.906310:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574083.906314:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574083.906327:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574083.906332:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14550 00000400:00000200:0.0:1788574083.906373:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574083.906380:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574083.906383:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574083.906387:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906390:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906395:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906398:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574083.906402:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574083.906405:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574083.906407:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574083.906409:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574083.906410:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574083.906412:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574083.906415:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906416:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906417:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906418:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574083.906421:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574083.906427:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906431:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574083.906435:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574083.906440:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574083.906443:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906452:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574083.906456:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574083.906459:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906460:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574083.906465:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574083.906474:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574083.906478:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574083.906481:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906484:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574083.906486:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574083.906491:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574083.906498:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574083.906501:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574083.906508:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810213760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574083.906516:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574083.906520:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.906528:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810213760/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574083.906551:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574083.906555:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574083.906563:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574083.906568:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574083.906573:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574083.906581:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574083.906588:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14550 00000400:00000200:0.0:1788574083.906599:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574083.906606:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574083.906611:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574083.906617:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906623:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906628:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906632:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574083.906670:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574083.906673:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574083.906683:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574083.906688:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574083.906691:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574083.906696:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574083.906703:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906708:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.906713:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906719:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574083.906725:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574083.906735:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574083.906748:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574083.906750:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574083.906755:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574083.906760:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906762:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574083.906766:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574083.906773:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906777:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574083.906787:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574083.906794:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574083.906813:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574083.906816:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.906821:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574083.906826:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574083.906831:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574083.906837:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574083.906844:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574083.906856:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810213888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574083.906883:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574083.906889:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.906897:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810213888/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574083.906924:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574083.906930:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574083.906939:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574083.906941:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574083.906943:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574083.906947:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574083.906954:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14550 00000400:00000200:0.0:1788574083.906965:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574083.906974:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574083.906981:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574083.907006:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.907012:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.907016:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.907020:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574083.907026:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574083.907027:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574083.907031:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574083.907034:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574083.907041:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574083.907047:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574083.907055:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.907061:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574083.907067:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574083.907072:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574083.907077:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574083.907085:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574083.907091:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574083.907104:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574083.907109:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574083.907116:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.907121:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574083.907126:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574083.907132:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.907136:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574083.907146:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574083.907155:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574083.907161:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574083.907165:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574083.907170:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574083.907174:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574083.907180:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574083.907189:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574083.907198:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574083.907210:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810214016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574083.907232:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574083.907235:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.907241:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810214016/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574083.907257:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.907267:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14560) 00000020:00000001:0.0:1788574086.337801:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574086.337804:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574086.337806:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574087.745736:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309226718, at 4309226752 00000100:00000010:0.0:1788574087.745750:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788574091.457858:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574091.457863:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574091.457868:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574092.865714:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309231838, at 4309231872 00000100:00000010:0.0:1788574092.865741:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 10000000:00000001:0.0:1788574093.633772:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574093.633790:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9600. 00000020:00000010:0.0:1788574093.633798:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574093.633805:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574093.633809:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574093.633814:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574093.633820:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574093.633824:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574093.633829:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574093.633833:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788574093.633837:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574093.633840:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574093.633845:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574093.633852:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574093.633854:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574093.633858:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574093.633860:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574093.633863:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574093.633865:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574093.633867:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574093.633869:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9600. 10000000:01000000:0.0:1788574093.633872:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574093.633875:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574093.633877:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574093.633879:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574093.633880:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574093.633882:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574093.633884:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574093.633886:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574093.633888:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574093.633890:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574093.633892:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574093.633926:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574093.633928:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574095.425768:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574095.425780:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574095.425785:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574095.425788:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574095.425790:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574095.425792:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574095.425795:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574095.425797:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574095.425800:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574095.425803:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574095.425806:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574095.425820:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574095.425825:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574095.425838:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.425842:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.425849:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.425859:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574095.425867:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574095.425871:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574095.425874:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574095.425883:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692300. 00010000:00000001:0.0:1788574095.425887:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574095.425891:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788574095.425895:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574095.425929:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168bae to hash 00000020:00000001:0.0:1788574095.425932:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574095.425934:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574095.425938:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574095.425943:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574095.425947:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574095.425952:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574095.425963:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574095.426002:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574095.426010:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574095.426013:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574095.426021:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574095.426024:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574095.426026:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574095.426028:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788574095.426033:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.426037:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.426039:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574095.426041:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574095.426052:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.426059:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574095.426063:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.426068:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.426071:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.426075:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574095.426077:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574095.426078:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.426080:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574095.426082:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574095.426085:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574095.426088:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574095.426090:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574095.426092:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574095.426101:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574095.426107:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810215424:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574095.426112:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574095.426114:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574095.426116:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574095.426118:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574095.426121:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.426126:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.426128:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574095.426134:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574095.426135:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574095.426137:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574095.426141:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574095.426143:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.426145:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788574095.426146:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574095.426149:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788574095.426152:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426157:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574095.426160:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426164:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574095.426166:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426178:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574095.426179:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574095.426181:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426185:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574095.426186:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574095.426189:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574095.426191:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574095.426195:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574095.426197:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574095.426199:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426203:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574095.426204:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574095.426206:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.426208:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788574095.426210:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574095.426212:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574095.426216:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574095.426219:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574095.426227:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574095.426229:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574095.426236:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574095.426244:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574095.426246:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574095.426250:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574095.426252:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574095.426256:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574095.426258:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574095.426261:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574095.426263:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574095.426265:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168bae from hash 00010000:00000001:0.0:1788574095.426267:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574095.426268:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574095.426270:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574095.426271:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574095.426272:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574095.426273:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574095.426276:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574095.426276:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574095.426277:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574095.426278:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574095.426280:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574095.426284:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574095.426285:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574095.426286:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574095.426287:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574095.426288:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574095.426289:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574095.426290:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574095.426292:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bae lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574095.426298:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692300 count: 0 00010000:00000001:0.0:1788574095.426307:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574095.426309:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574095.426310:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574095.426318:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810215424/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.426323:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574095.426326:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574095.426328:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574095.426329:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574095.426332:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574095.426336:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574095.426337:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574095.426339:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788574095.426342:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574095.426343:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574095.426345:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574095.426347:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574095.426350:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574096.577771:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574096.577776:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574096.577779:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574099.265735:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309238239, at 4309238272 00000100:00000010:0.0:1788574099.265754:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574101.697902:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574101.697906:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574101.697909:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574103.810018:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574104.641728:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309243614, at 4309243648 00000100:00000010:0.0:1788574104.641744:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788574106.817874:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574106.817878:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574106.817881:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574109.761725:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309248734, at 4309248768 00000100:00000010:0.0:1788574109.761740:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00080000:0.0:1788574111.041739:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574111.041747:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574111.041764:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574111.041767:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574111.041771:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574111.041782:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14577 00000400:00000200:0.0:1788574111.041789:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574111.041794:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574111.041797:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574111.041799:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041802:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.041804:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041806:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574111.041809:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574111.041811:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574111.041813:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574111.041816:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574111.041818:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574111.041820:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574111.041822:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041823:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.041825:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041826:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574111.041828:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574111.041831:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041833:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574111.041835:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574111.041837:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574111.041840:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.041841:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574111.041843:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574111.041849:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574111.041851:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574111.041869:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574111.041871:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.041874:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574111.041876:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574111.041880:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574111.041886:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574111.041892:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.041899:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810217472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574111.041907:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574111.041911:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.041914:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810217472/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574111.041932:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574111.041934:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574111.041937:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574111.041938:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574111.041939:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574111.041941:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574111.041943:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14577 00000400:00000200:0.0:1788574111.041946:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574111.041948:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574111.041949:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574111.041952:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041953:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.041955:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041957:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574111.041959:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574111.041960:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574111.041961:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574111.041962:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574111.041967:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574111.041969:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574111.041973:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041977:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.041979:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041984:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574111.041986:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574111.041990:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574111.041992:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574111.041993:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574111.041999:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574111.042004:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042008:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574111.042013:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574111.042017:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042019:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574111.042022:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574111.042026:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574111.042032:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574111.042033:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042037:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574111.042039:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574111.042042:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574111.042048:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574111.042053:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.042062:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810217600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574111.042070:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574111.042074:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.042079:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810217600/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574111.042104:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574111.042109:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574111.042111:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574111.042120:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574111.042125:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574111.042131:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574111.042136:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14577 00000400:00000200:0.0:1788574111.042142:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574111.042148:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574111.042152:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574111.042158:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042163:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042168:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042170:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574111.042175:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574111.042178:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574111.042180:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574111.042182:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574111.042185:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574111.042189:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574111.042193:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042195:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042196:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042198:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574111.042199:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574111.042204:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042213:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574111.042225:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574111.042231:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574111.042235:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042238:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574111.042242:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574111.042249:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042253:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574111.042261:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574111.042265:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574111.042271:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574111.042275:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042281:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574111.042284:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574111.042290:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574111.042297:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574111.042305:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.042317:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810217728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574111.042335:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574111.042340:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.042349:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810217728/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574111.042372:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574111.042378:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574111.042386:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574111.042391:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574111.042395:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574111.042401:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574111.042405:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14577 00000400:00000200:0.0:1788574111.042415:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574111.042425:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574111.042428:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574111.042444:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042450:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042453:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042458:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574111.042464:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574111.042466:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574111.042469:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574111.042473:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574111.042476:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574111.042479:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574111.042485:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042488:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042494:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042498:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574111.042503:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574111.042508:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042514:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574111.042518:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574111.042529:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574111.042533:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042538:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574111.042541:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574111.042546:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042550:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574111.042554:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574111.042558:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574111.042564:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574111.042567:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042570:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574111.042573:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574111.042576:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574111.042581:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574111.042604:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.042611:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810217856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574111.042618:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574111.042622:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.042628:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810217856/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574111.042666:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574111.042671:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574111.042676:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574111.042679:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574111.042683:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574111.042686:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574111.042689:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14577 00000400:00000200:0.0:1788574111.042693:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574111.042699:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574111.042704:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574111.042711:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042716:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042722:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042724:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574111.042728:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574111.042729:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574111.042732:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574111.042736:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574111.042741:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574111.042745:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574111.042752:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042754:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574111.042763:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042767:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574111.042771:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574111.042775:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574111.042778:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574111.042782:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574111.042787:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574111.042790:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042793:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574111.042797:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574111.042801:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042804:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574111.042808:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574111.042813:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574111.042817:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574111.042819:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574111.042823:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574111.042825:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574111.042829:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574111.042835:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574111.042842:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.042848:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810217984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574111.042862:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574111.042866:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.042872:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810217984/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574111.042889:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574111.042894:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14587) 00000020:00000001:0.0:1788574111.937785:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574111.937790:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574111.937793:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574114.881783:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309253854, at 4309253888 00000100:00000010:0.0:1788574114.881798:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 10000000:00000001:0.0:1788574117.057794:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574117.057800:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574117.057807:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574117.057810:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574117.057811:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574117.057813:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574117.057817:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574117.057819:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574117.057824:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574117.057827:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574117.057832:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574117.057844:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574117.057848:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574117.057853:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574117.057856:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574117.057860:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574117.057866:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574117.057873:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574117.057877:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574117.057880:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574117.057886:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692300. 00010000:00000001:0.0:1788574117.057890:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574117.057892:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788574117.057896:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574117.057898:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168bb5 to hash 00000020:00000001:0.0:1788574117.057899:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574117.057900:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574117.057903:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574117.057905:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574117.057909:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574117.057914:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574117.057950:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574117.057963:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574117.057968:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574117.057970:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574117.057975:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574117.057977:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574117.057979:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574117.057980:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000001:0.0:1788574117.057984:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574117.057988:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574117.057990:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574117.057991:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574117.058000:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574117.058006:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574117.058009:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574117.058014:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574117.058016:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574117.058021:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574117.058023:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574117.058024:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574117.058025:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574117.058027:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574117.058029:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574117.058036:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.058037:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574117.058039:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574117.058042:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574117.058048:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810218752:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574117.058052:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.058054:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574117.058056:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574117.058057:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574117.058060:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574117.058064:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574117.058066:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574117.058070:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574117.058071:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574117.058073:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574117.058076:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574117.058078:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574117.058080:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788574117.058082:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574117.058085:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788574117.058088:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058096:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574117.058100:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058107:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574117.058115:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058122:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574117.058124:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574117.058126:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058130:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574117.058132:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574117.058134:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574117.058136:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574117.058140:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574117.058141:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574117.058143:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058147:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574117.058149:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574117.058150:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574117.058152:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788574117.058154:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574117.058156:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574117.058159:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574117.058162:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574117.058169:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574117.058172:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574117.058178:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574117.058186:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574117.058188:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574117.058191:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574117.058193:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574117.058196:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574117.058198:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574117.058199:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574117.058200:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574117.058202:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168bb5 from hash 00010000:00000001:0.0:1788574117.058205:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574117.058206:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574117.058207:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574117.058207:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574117.058209:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574117.058210:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574117.058212:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574117.058213:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574117.058214:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574117.058215:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574117.058225:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574117.058232:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574117.058233:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574117.058234:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574117.058235:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574117.058236:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574117.058237:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574117.058237:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574117.058239:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168bb5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574117.058244:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692300 count: 0 00010000:00000001:0.0:1788574117.058251:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574117.058252:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574117.058254:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574117.058261:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7480 x1875440810218752/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574117.058266:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574117.058267:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574117.058269:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574117.058270:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574117.058272:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574117.058275:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574117.058277:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574117.058278:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7480. 00000100:00000001:0.0:1788574117.058282:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574117.058282:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574117.058284:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574117.058285:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574117.058288:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574120.897842:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309259871, at 4309259904 00000100:00000010:0.0:1788574120.897847:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00080000:0.0:1788574122.305733:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574122.305744:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574122.305747:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574122.305751:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574122.305756:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574122.305763:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14588 00000400:00000200:0.0:1788574122.305773:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574122.305779:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574122.305783:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574122.305786:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305789:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.305791:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305800:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574122.305804:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574122.305805:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574122.305823:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574122.305826:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574122.305828:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574122.305830:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574122.305832:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305834:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.305835:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305837:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574122.305839:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574122.305842:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305843:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574122.305846:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574122.305849:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574122.305851:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.305852:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574122.305854:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574122.305860:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574122.305863:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574122.305867:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574122.305869:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.305872:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574122.305875:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574122.305880:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574122.305889:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574122.305897:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.305907:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810219520/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574122.305917:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574122.305920:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.305923:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810219520/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574122.305944:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574122.305951:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574122.305954:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574122.305955:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574122.305957:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574122.305959:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574122.305961:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14588 00000400:00000200:0.0:1788574122.305965:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574122.305969:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574122.305971:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574122.305974:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305978:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.305981:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.305985:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574122.305988:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574122.305989:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574122.305992:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574122.306000:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574122.306004:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574122.306007:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574122.306012:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306016:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306021:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306026:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574122.306030:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574122.306037:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306042:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574122.306044:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574122.306051:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574122.306054:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306056:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574122.306065:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574122.306071:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306074:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574122.306080:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574122.306092:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574122.306099:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574122.306104:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306110:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574122.306114:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574122.306119:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574122.306129:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574122.306138:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.306146:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810219648/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574122.306155:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574122.306158:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.306174:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810219648/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574122.306199:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574122.306206:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574122.306214:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574122.306219:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574122.306222:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574122.306229:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574122.306235:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14588 00000400:00000200:0.0:1788574122.306258:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574122.306262:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574122.306263:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574122.306266:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306267:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306269:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306278:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574122.306280:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574122.306282:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574122.306283:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574122.306284:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574122.306286:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574122.306288:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574122.306291:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306294:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306299:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306304:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574122.306311:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574122.306321:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306329:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574122.306341:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574122.306346:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574122.306350:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306353:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574122.306357:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574122.306360:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306364:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574122.306368:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574122.306373:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574122.306380:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574122.306385:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306391:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574122.306396:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574122.306401:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574122.306408:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574122.306417:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.306426:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810219776/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574122.306460:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574122.306466:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.306476:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810219776/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574122.306491:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574122.306497:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574122.306501:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574122.306503:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574122.306507:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574122.306512:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574122.306517:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14588 00000400:00000200:0.0:1788574122.306524:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574122.306531:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574122.306536:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574122.306543:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306547:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306552:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306557:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574122.306561:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574122.306564:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574122.306567:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574122.306570:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574122.306576:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574122.306579:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574122.306584:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306590:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306594:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306599:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574122.306607:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574122.306611:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306615:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574122.306618:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574122.306623:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574122.306627:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306632:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574122.306678:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574122.306681:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306683:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574122.306687:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574122.306692:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574122.306696:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574122.306699:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306704:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574122.306707:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574122.306710:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574122.306716:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574122.306720:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.306731:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810219904/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574122.306759:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574122.306765:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.306779:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810219904/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574122.306794:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574122.306799:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574122.306802:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574122.306806:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574122.306811:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574122.306813:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574122.306824:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14588 00000400:00000200:0.0:1788574122.306833:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574122.306836:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574122.306838:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574122.306843:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306850:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306856:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306860:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574122.306863:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574122.306864:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574122.306869:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574122.306871:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574122.306877:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574122.306883:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574122.306893:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306899:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574122.306907:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306913:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574122.306917:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574122.306922:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574122.306926:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574122.306932:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574122.306936:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574122.306939:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306942:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574122.306946:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574122.306953:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306954:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574122.306958:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574122.306961:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574122.306964:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574122.306971:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574122.306974:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574122.306977:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574122.306980:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574122.306985:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574122.306991:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.307000:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810220032/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574122.307009:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574122.307013:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.307021:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810220032/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574122.307032:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574122.307038:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14599) 00000100:00000040:0.0:1788574126.145722:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309265118, at 4309265152 00000100:00000010:0.0:1788574126.145736:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00080000:0.0:1788574127.425758:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574127.425770:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574127.425773:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574127.425777:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574127.425783:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574127.425790:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14594 00000400:00000200:0.0:1788574127.425800:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574127.425806:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574127.425809:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574127.425812:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425815:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.425816:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425818:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574127.425822:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574127.425823:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574127.425841:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574127.425844:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574127.425845:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574127.425847:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574127.425849:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425855:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.425857:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425858:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574127.425860:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574127.425868:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425870:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574127.425873:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574127.425876:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574127.425878:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.425880:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574127.425882:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574127.425887:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574127.425890:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574127.425893:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574127.425895:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.425897:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574127.425900:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574127.425904:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574127.425912:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574127.425917:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.425924:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810220160/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574127.425932:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574127.425935:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.425939:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810220160/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574127.425957:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574127.425964:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574127.425966:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574127.425967:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574127.425969:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574127.425971:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574127.425973:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14594 00000400:00000200:0.0:1788574127.425976:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574127.425978:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574127.425980:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574127.425982:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425984:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.425986:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.425988:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574127.425990:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574127.425991:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574127.425992:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574127.425994:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574127.425995:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574127.425997:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574127.425999:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426000:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426002:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426004:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574127.426008:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574127.426010:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426015:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574127.426016:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574127.426023:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574127.426026:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426027:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574127.426034:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574127.426038:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426043:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574127.426048:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574127.426052:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574127.426057:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574127.426061:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426064:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574127.426067:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574127.426070:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574127.426075:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574127.426081:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.426103:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810220288/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574127.426110:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574127.426111:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.426115:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810220288/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574127.426124:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574127.426130:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574127.426137:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574127.426139:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574127.426143:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574127.426149:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574127.426154:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14594 00000400:00000200:0.0:1788574127.426163:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574127.426171:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574127.426175:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574127.426183:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426185:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426187:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426198:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574127.426201:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574127.426204:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574127.426207:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574127.426211:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574127.426215:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574127.426219:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574127.426224:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426227:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426230:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426235:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574127.426237:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574127.426242:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426245:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574127.426247:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574127.426249:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574127.426254:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426257:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574127.426259:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574127.426263:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426266:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574127.426270:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574127.426274:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574127.426278:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574127.426280:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426283:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574127.426285:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574127.426288:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574127.426294:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574127.426301:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.426308:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810220416/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574127.426335:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574127.426338:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.426345:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810220416/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574127.426360:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574127.426365:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574127.426368:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574127.426369:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574127.426371:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574127.426373:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574127.426378:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14594 00000400:00000200:0.0:1788574127.426383:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574127.426386:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574127.426388:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574127.426390:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426395:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426397:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426399:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574127.426401:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574127.426401:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574127.426403:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574127.426404:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574127.426405:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574127.426407:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574127.426410:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426414:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426418:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426422:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574127.426429:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574127.426436:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426440:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574127.426444:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574127.426447:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574127.426451:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426456:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574127.426459:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574127.426463:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426466:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574127.426471:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574127.426475:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574127.426481:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574127.426483:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426486:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574127.426488:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574127.426492:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574127.426496:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574127.426499:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.426506:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810220544/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574127.426518:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574127.426520:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.426524:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810220544/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574127.426532:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574127.426534:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574127.426539:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574127.426543:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574127.426545:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574127.426550:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574127.426559:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14594 00000400:00000200:0.0:1788574127.426566:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574127.426571:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574127.426575:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574127.426580:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426585:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426590:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426595:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574127.426600:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574127.426604:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574127.426608:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574127.426611:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574127.426619:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574127.426622:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574127.426626:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426630:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574127.426664:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426667:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574127.426672:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574127.426679:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574127.426686:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574127.426688:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574127.426693:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574127.426699:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426704:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574127.426709:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574127.426716:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426719:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574127.426725:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574127.426730:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574127.426736:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574127.426745:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574127.426749:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574127.426753:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574127.426758:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574127.426768:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574127.426777:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.426788:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810220672/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574127.426802:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574127.426806:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.426813:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810220672/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574127.426821:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574127.426828:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14604) 00000100:00000040:0.0:1788574131.265781:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309270238, at 4309270272 00000100:00000010:0.0:1788574131.265797:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000100:00000040:0.0:1788574136.385729:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309275358, at 4309275392 00000100:00000010:0.0:1788574136.385748:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 10000000:00000001:0.0:1788574137.153763:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574137.153778:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9600. 00000020:00000010:0.0:1788574137.153786:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574137.153803:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574137.153808:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574137.153813:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574137.153818:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574137.153823:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574137.153827:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574137.153831:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788574137.153836:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574137.153839:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574137.153845:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574137.153852:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574137.153868:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574137.153872:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574137.153875:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574137.153880:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574137.153883:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574137.153886:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574137.153889:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9600. 10000000:01000000:0.0:1788574137.153892:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574137.153896:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574137.153899:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574137.153901:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574137.153903:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574137.153906:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574137.153909:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574137.153913:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574137.153914:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574137.153917:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574137.153920:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574137.153922:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574137.153925:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574138.753790:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574138.753798:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574138.753803:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574138.753806:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574138.753807:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574138.753809:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574138.753812:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574138.753814:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574138.753817:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574138.753820:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574138.753822:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574138.753834:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574138.753839:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574138.753844:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574138.753848:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574138.753854:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574138.753862:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574138.753888:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574138.753892:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574138.753895:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574138.753903:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692300. 00010000:00000001:0.0:1788574138.753908:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574138.753912:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788574138.753917:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574138.753921:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168bbc to hash 00000020:00000001:0.0:1788574138.753923:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574138.753925:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574138.753928:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574138.753932:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574138.753936:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574138.753941:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574138.753953:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574138.753966:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574138.753974:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574138.753976:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574138.754018:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574138.754021:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574138.754024:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574138.754026:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531983232 : -109714177568384 : ffff9c372d6b5180) 00000100:00000001:0.0:1788574138.754032:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574138.754038:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574138.754040:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574138.754042:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574138.754062:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574138.754084:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574138.754090:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574138.754096:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574138.754101:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574138.754108:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574138.754111:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574138.754113:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574138.754116:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574138.754119:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574138.754122:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574138.754126:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574138.754129:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574138.754131:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574138.754136:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574138.754144:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810222080:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574138.754150:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574138.754153:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574138.754156:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574138.754158:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574138.754161:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574138.754169:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574138.754172:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574138.754180:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574138.754182:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574138.754185:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574138.754199:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574138.754201:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574138.754204:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788574138.754206:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574138.754209:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788574138.754212:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754219:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574138.754224:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754230:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574138.754233:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754241:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574138.754242:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574138.754245:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754251:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574138.754253:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574138.754256:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574138.754259:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574138.754266:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574138.754269:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574138.754271:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754289:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574138.754291:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574138.754292:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574138.754294:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788574138.754296:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574138.754298:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574138.754302:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574138.754304:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574138.754311:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574138.754313:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574138.754320:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574138.754322:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574138.754324:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574138.754327:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574138.754330:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574138.754334:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574138.754336:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574138.754338:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574138.754340:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574138.754343:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168bbc from hash 00010000:00000001:0.0:1788574138.754346:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574138.754347:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574138.754348:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574138.754350:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574138.754351:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574138.754354:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574138.754357:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574138.754358:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574138.754360:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574138.754366:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574138.754370:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574138.754388:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574138.754390:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574138.754392:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574138.754393:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574138.754395:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574138.754396:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574138.754397:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574138.754400:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bbc lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574138.754408:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692300 count: 0 00010000:00000001:0.0:1788574138.754420:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574138.754422:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574138.754426:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574138.754431:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5180 x1875440810222080/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574138.754439:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574138.754442:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574138.754444:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574138.754446:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574138.754450:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574138.754455:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574138.754457:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574138.754460:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5180. 00000100:00000001:0.0:1788574138.754464:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574138.754466:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574138.754468:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574138.754471:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574138.754474:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574142.593744:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309281567, at 4309281600 00000100:00000010:0.0:1788574142.593761:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000800:00000200:0.0:1788574142.721869:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574147.649768:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309286622, at 4309286656 00000100:00000010:0.0:1788574147.649788:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00000040:0.0:1788574152.769761:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309291742, at 4309291776 00000100:00000010:0.0:1788574152.769782:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00000040:0.0:1788574157.889706:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309296862, at 4309296896 00000100:00000010:0.0:1788574157.889720:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 10000000:00000001:0.0:1788574159.169743:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574159.169759:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9fc0. 00000020:00000010:0.0:1788574159.169766:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574159.169771:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574159.169774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574159.169777:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574159.169782:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574159.169785:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574159.169788:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574159.169792:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788574159.169796:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574159.169798:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574159.169802:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574159.169807:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574159.169809:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574159.169812:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574159.169814:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574159.169816:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574159.169818:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574159.169820:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574159.169822:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9fc0. 10000000:01000000:0.0:1788574159.169824:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574159.169827:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574159.169829:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574159.169843:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574159.169844:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574159.169847:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574159.169848:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574159.169851:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574159.169852:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574159.169855:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574159.169857:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574159.169858:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574159.169860:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788574159.169877:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574159.169881:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574159.169883:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574159.169885:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574159.169888:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574159.169894:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14625 00000400:00000200:0.0:1788574159.169900:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574159.169905:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574159.169908:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574159.169910:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.169913:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.169915:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.169916:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574159.169919:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574159.169920:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574159.169922:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574159.169930:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574159.169932:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574159.169933:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574159.169935:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.169937:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.169938:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.169939:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574159.169946:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574159.169948:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574159.169950:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574159.169953:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574159.169956:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574159.169958:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.169959:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574159.169961:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574159.169966:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574159.169968:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574159.169972:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574159.169973:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.169975:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574159.169978:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574159.169982:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574159.169988:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574159.169995:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.170001:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810224768/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574159.170009:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574159.170013:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170017:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810224768/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574159.170034:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574159.170036:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574159.170039:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574159.170040:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574159.170041:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574159.170043:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574159.170045:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14625 00000400:00000200:0.0:1788574159.170047:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574159.170054:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574159.170056:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574159.170058:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170059:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170061:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170063:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574159.170065:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574159.170066:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574159.170067:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574159.170068:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574159.170070:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574159.170072:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574159.170074:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170076:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170077:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170079:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574159.170082:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574159.170086:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170088:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574159.170091:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574159.170096:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574159.170100:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170102:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574159.170106:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574159.170112:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170116:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574159.170120:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574159.170139:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574159.170143:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574159.170144:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170146:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574159.170154:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574159.170157:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574159.170162:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574159.170167:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.170174:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810224896/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574159.170186:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574159.170189:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170196:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810224896/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574159.170209:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574159.170212:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574159.170215:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574159.170219:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574159.170221:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574159.170224:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574159.170229:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14625 00000400:00000200:0.0:1788574159.170235:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574159.170239:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574159.170244:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574159.170246:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170249:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170254:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170257:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574159.170260:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574159.170264:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574159.170269:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574159.170273:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574159.170279:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574159.170284:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574159.170301:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170306:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170311:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170314:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574159.170316:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574159.170322:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170333:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574159.170334:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574159.170337:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574159.170342:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170345:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574159.170348:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574159.170350:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170352:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574159.170357:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574159.170361:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574159.170363:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574159.170366:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170369:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574159.170370:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574159.170374:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574159.170380:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574159.170385:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.170393:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810225024/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574159.170399:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574159.170401:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170403:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810225024/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574159.170414:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574159.170419:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574159.170428:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574159.170429:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574159.170431:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574159.170437:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574159.170442:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14625 00000400:00000200:0.0:1788574159.170458:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574159.170465:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574159.170469:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574159.170472:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170477:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170479:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170480:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574159.170482:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574159.170483:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574159.170486:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574159.170488:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574159.170494:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574159.170496:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574159.170502:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170504:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170509:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170514:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574159.170518:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574159.170523:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170526:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574159.170528:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574159.170532:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574159.170536:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170538:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574159.170542:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574159.170551:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170554:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574159.170559:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574159.170562:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574159.170564:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574159.170567:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170570:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574159.170572:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574159.170575:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574159.170579:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574159.170583:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.170587:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810225152/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574159.170594:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574159.170596:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170604:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810225152/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574159.170623:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574159.170626:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574159.170629:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574159.170631:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574159.170659:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574159.170661:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574159.170665:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14625 00000400:00000200:0.0:1788574159.170673:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574159.170680:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574159.170685:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574159.170690:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170693:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170697:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170707:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574159.170714:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574159.170718:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574159.170721:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574159.170726:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574159.170730:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574159.170735:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574159.170741:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170745:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574159.170747:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170749:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574159.170753:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574159.170757:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574159.170761:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574159.170763:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574159.170767:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574159.170770:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170775:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574159.170778:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574159.170785:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170786:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574159.170794:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574159.170797:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574159.170799:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574159.170802:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.170804:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574159.170807:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574159.170809:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574159.170814:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574159.170819:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.170827:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810225280/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574159.170866:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574159.170871:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170880:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810225280/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574159.170895:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574159.170903:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14635) 10000000:00000001:0.0:1788574160.769773:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574160.769783:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574160.769788:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574160.769791:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574160.769792:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574160.769794:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574160.769797:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574160.769799:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574160.769801:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574160.769804:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574160.769806:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574160.769817:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574160.769823:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574160.769828:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574160.769831:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574160.769838:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574160.769849:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574160.769856:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574160.769860:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574160.769863:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574160.769872:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692000. 00010000:00000001:0.0:1788574160.769877:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574160.769881:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788574160.769887:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574160.769892:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x1441586093168bc3 to hash 00000020:00000001:0.0:1788574160.769894:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574160.769896:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574160.769900:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574160.769924:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574160.769928:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574160.769933:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574160.769946:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574160.769960:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574160.769967:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574160.769970:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574160.769979:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574160.769982:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574160.769986:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574160.769988:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574160.769994:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574160.770000:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574160.770003:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574160.770005:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574160.770019:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574160.770030:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574160.770036:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574160.770043:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574160.770046:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574160.770052:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574160.770054:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574160.770062:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574160.770065:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574160.770067:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574160.770070:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574160.770084:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.770088:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574160.770090:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574160.770094:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574160.770101:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810225408:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574160.770106:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.770108:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574160.770110:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574160.770112:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574160.770116:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574160.770124:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574160.770127:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574160.770133:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574160.770135:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574160.770138:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574160.770150:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574160.770153:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574160.770155:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=4 00000020:00000001:0.0:1788574160.770158:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574160.770162:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00010000:0.0:1788574160.770166:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770173:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574160.770176:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770187:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574160.770190:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770194:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574160.770196:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574160.770197:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770201:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574160.770202:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574160.770205:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574160.770207:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574160.770211:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574160.770213:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574160.770215:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770219:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574160.770221:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574160.770222:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574160.770223:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=5 00000020:00000001:0.0:1788574160.770224:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574160.770226:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574160.770228:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574160.770231:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574160.770235:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574160.770239:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574160.770244:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574160.770246:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574160.770247:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574160.770249:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574160.770251:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574160.770254:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574160.770255:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574160.770256:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574160.770257:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574160.770260:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31400 with handle 0x1441586093168bc3 from hash 00010000:00000001:0.0:1788574160.770262:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574160.770263:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574160.770264:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574160.770265:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574160.770266:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574160.770268:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574160.770269:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574160.770270:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574160.770271:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574160.770272:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574160.770274:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574160.770277:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574160.770278:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574160.770279:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574160.770280:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574160.770281:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574160.770282:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574160.770282:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574160.770284:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168bc3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574160.770289:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692000 count: 0 00010000:00000001:0.0:1788574160.770303:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574160.770304:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574160.770306:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574160.770309:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810225408/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574160.770314:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574160.770317:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574160.770318:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574160.770320:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574160.770323:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574160.770327:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574160.770329:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574160.770330:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574160.770333:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574160.770334:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574160.770336:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574160.770338:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574160.770341:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574164.609715:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309303583, at 4309303616 00000100:00000010:0.0:1788574164.609730:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00000040:0.0:1788574169.665810:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309308639, at 4309308672 00000100:00000010:0.0:1788574169.665829:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00000040:0.0:1788574174.785786:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309313759, at 4309313792 00000100:00000010:0.0:1788574174.785817:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00000040:0.0:1788574179.905738:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309318878, at 4309318912 00000100:00000010:0.0:1788574179.905760:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000800:00000200:0.0:1788574180.609902:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 10000000:00000001:0.0:1788574181.185764:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574181.185778:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9c00. 00000020:00000010:0.0:1788574181.185787:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574181.185804:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574181.185811:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574181.185820:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574181.185839:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574181.185846:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574181.185851:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574181.185858:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788574181.185863:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574181.185867:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574181.185873:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574181.185880:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574181.185884:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574181.185888:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574181.185892:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574181.185896:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b400. 20000000:00000010:0.0:1788574181.185898:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574181.185900:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574181.185905:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9c00. 10000000:01000000:0.0:1788574181.185908:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574181.185910:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574181.185916:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574181.185917:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574181.185918:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574181.185922:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574181.185924:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574181.185929:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574181.185931:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574181.185935:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574181.185939:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574181.185941:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574181.185945:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574182.273789:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574182.273797:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574182.273803:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574182.273806:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574182.273808:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574182.273810:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574182.273812:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574182.273830:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574182.273832:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574182.273835:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574182.273837:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574182.273845:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574182.273851:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574182.273856:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574182.273859:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574182.273865:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574182.273873:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574182.273882:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574182.273886:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574182.273889:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574182.273898:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692000. 00010000:00000001:0.0:1788574182.273902:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574182.273907:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30c00. 00000020:00000001:0.0:1788574182.273911:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574182.273914:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30c00 with handle 0x1441586093168bca to hash 00000020:00000001:0.0:1788574182.273916:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574182.273918:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788574182.273922:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788574182.273925:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574182.273938:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574182.273942:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574182.273953:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574182.273966:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574182.273972:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574182.273974:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574182.273987:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574182.273989:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574182.273992:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574182.273994:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574182.273999:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574182.274025:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574182.274028:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574182.274030:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574182.274040:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574182.274050:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574182.274054:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574182.274062:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574182.274066:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574182.274074:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574182.274077:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574182.274079:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574182.274089:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574182.274092:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574182.274095:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574182.274099:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574182.274101:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574182.274103:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574182.274108:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574182.274116:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810228736:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574182.274122:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574182.274124:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574182.274132:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574182.274135:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574182.274140:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574182.274153:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574182.274155:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574182.274161:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574182.274162:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574182.274165:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574182.274168:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574182.274171:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574182.274173:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=4 00000020:00000001:0.0:1788574182.274176:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788574182.274179:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00010000:0.0:1788574182.274182:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274190:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574182.274194:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274206:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574182.274209:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274216:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574182.274218:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574182.274221:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274227:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574182.274229:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574182.274237:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574182.274241:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574182.274249:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30c00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574182.274251:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574182.274255:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274261:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574182.274263:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574182.274264:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574182.274266:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30c00 refcount=5 00000020:00000001:0.0:1788574182.274268:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788574182.274270:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536684032 : -109714172867584 : ffff9c372db30c00) 00010000:00000001:0.0:1788574182.274273:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574182.274276:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574182.274283:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574182.274286:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574182.274293:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574182.274296:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574182.274298:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574182.274301:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574182.274309:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574182.274312:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574182.274313:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574182.274315:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574182.274317:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574182.274320:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30c00 with handle 0x1441586093168bca from hash 00010000:00000001:0.0:1788574182.274322:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574182.274328:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574182.274329:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574182.274330:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574182.274332:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574182.274334:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574182.274337:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574182.274338:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574182.274339:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574182.274342:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574182.274345:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574182.274351:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574182.274352:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574182.274354:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574182.274356:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574182.274357:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574182.274359:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574182.274360:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574182.274363:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30c00/0x1441586093168bca lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574182.274371:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692000 count: 0 00010000:00000001:0.0:1788574182.274380:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574182.274382:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574182.274384:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574182.274389:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810228736/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574182.274396:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574182.274399:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574182.274401:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574182.274403:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574182.274406:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574182.274411:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574182.274413:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574182.274415:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574182.274423:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574182.274425:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574182.274427:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574182.274429:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574182.274432:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574186.113705:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309325087, at 4309325120 00000100:00000010:0.0:1788574186.113727:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00000040:0.0:1788574191.169725:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309330142, at 4309330176 00000100:00000010:0.0:1788574191.169739:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00000040:0.0:1788574196.289761:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309335262, at 4309335296 00000100:00000010:0.0:1788574196.289780:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788574201.409710:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309340382, at 4309340416 00000100:00000010:0.0:1788574201.409728:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215120. 10000000:00000001:0.0:1788574202.689765:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574202.689778:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9ec0. 00000020:00000010:0.0:1788574202.689785:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574202.689794:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574202.689800:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574202.689807:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574202.689814:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574202.689821:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574202.689826:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574202.689846:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788574202.689854:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574202.689856:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574202.689861:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574202.689870:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574202.689875:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574202.689883:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788574202.689888:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574202.689892:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574202.689899:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574202.689915:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574202.689921:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9ec0. 10000000:01000000:0.0:1788574202.689928:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574202.689934:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574202.689939:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574202.689941:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574202.689946:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574202.689950:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574202.689953:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574202.689957:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574202.689959:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574202.689966:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574202.689971:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574202.689976:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574202.689983:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574204.609763:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574204.609772:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574204.609778:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574204.609781:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574204.609783:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574204.609784:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574204.609787:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574204.609789:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574204.609792:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574204.609795:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574204.609797:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574204.609809:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574204.609815:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574204.609820:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574204.609823:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574204.609830:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574204.609838:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574204.609846:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574204.609850:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574204.609853:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574204.609861:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692800. 00010000:00000001:0.0:1788574204.609865:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574204.609883:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788574204.609889:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574204.609892:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x1441586093168bd1 to hash 00000020:00000001:0.0:1788574204.609895:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574204.609896:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574204.609900:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574204.609904:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574204.609908:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574204.609912:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574204.609923:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574204.609936:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574204.609943:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574204.609946:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574204.609954:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574204.609956:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611968 : -109714523939648 : ffff9c3718c61cc0) 00000100:00000001:0.0:1788574204.609959:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574204.609961:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574204.609966:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574204.609971:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574204.609973:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574204.609975:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574204.609986:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574204.609997:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574204.610002:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574204.610009:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574204.610021:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574204.610030:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574204.610033:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574204.610035:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574204.610038:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61cc0 going to sleep for 0 seconds 00000100:00000001:0.0:1788574204.610041:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574204.610043:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574204.610047:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574204.610050:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574204.610052:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574204.610056:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574204.610064:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810232064:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574204.610070:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574204.610073:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574204.610076:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574204.610078:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574204.610083:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574204.610091:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574204.610095:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61cc0. 00000100:00000001:0.0:1788574204.610101:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574204.610103:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574204.610106:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574204.610109:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574204.610111:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574204.610114:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788574204.610117:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574204.610120:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00010000:0.0:1788574204.610124:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610137:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574204.610142:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610146:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574204.610149:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610153:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574204.610154:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574204.610156:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610160:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574204.610161:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574204.610163:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574204.610165:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574204.610169:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574204.610170:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574204.610172:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610176:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574204.610177:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574204.610178:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574204.610179:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=5 00000020:00000001:0.0:1788574204.610180:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574204.610182:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574204.610187:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574204.610190:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574204.610194:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574204.610195:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574204.610199:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574204.610201:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574204.610203:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574204.610205:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574204.610208:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574204.610210:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574204.610212:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574204.610214:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574204.610215:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574204.610218:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30400 with handle 0x1441586093168bd1 from hash 00010000:00000001:0.0:1788574204.610221:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574204.610222:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574204.610224:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574204.610225:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574204.610226:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574204.610228:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574204.610230:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574204.610232:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574204.610234:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574204.610235:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574204.610239:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574204.610245:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574204.610246:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574204.610248:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574204.610250:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574204.610257:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574204.610258:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574204.610259:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574204.610262:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168bd1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574204.610270:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692800 count: 0 00010000:00000001:0.0:1788574204.610279:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574204.610281:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574204.610284:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574204.610289:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810232064/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574204.610297:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574204.610300:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574204.610301:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574204.610304:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574204.610307:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574204.610311:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574204.610313:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574204.610316:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574204.610320:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574204.610321:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574204.610324:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574204.610326:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574204.610330:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574208.449757:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309347423, at 4309347456 00000100:00000010:0.0:1788574208.449773:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215cc0. 00000100:00000040:0.0:1788574213.697756:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309352671, at 4309352704 00000100:00000010:0.0:1788574213.697776:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215a20. 00000800:00000200:0.0:1788574218.497934:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574218.817732:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309357790, at 4309357824 00000100:00000010:0.0:1788574218.817750:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215d80. 00000100:00000040:0.0:1788574223.937741:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309362911, at 4309362944 00000100:00000010:0.0:1788574223.937758:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322151e0. 10000000:00000001:0.0:1788574224.705712:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574224.705725:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9fc0. 00000020:00000010:0.0:1788574224.705732:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574224.705736:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574224.705739:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574224.705742:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574224.705746:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788574224.705749:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574224.705754:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788574224.705757:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788574224.705761:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574224.705763:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574224.705767:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574224.705773:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574224.705775:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574224.705778:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788574224.705780:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574224.705783:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788574224.705784:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574224.705786:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788574224.705788:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9fc0. 10000000:01000000:0.0:1788574224.705790:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574224.705793:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574224.705795:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574224.705796:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574224.705798:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574224.705800:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574224.705801:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574224.705804:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574224.705805:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574224.705807:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574224.705809:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574224.705810:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574224.705812:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574225.793756:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574225.793769:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574225.793774:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574225.793777:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574225.793779:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574225.793781:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574225.793784:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574225.793786:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574225.793788:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574225.793792:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574225.793794:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574225.793805:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574225.793812:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574225.793816:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574225.793819:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574225.793824:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574225.793830:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788574225.793836:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574225.793840:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574225.793842:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574225.793848:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692100. 00010000:00000001:0.0:1788574225.793852:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574225.793855:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788574225.793858:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574225.793861:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168bd8 to hash 00000020:00000001:0.0:1788574225.793863:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574225.793864:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574225.793866:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574225.793870:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574225.793872:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574225.793875:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574225.793884:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574225.793909:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574225.793913:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574225.793915:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574225.793920:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574225.793922:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611584 : -109714523940032 : ffff9c3718c61b40) 00000100:00000001:0.0:1788574225.793924:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574225.793925:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574225.793929:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574225.793932:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574225.793934:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574225.793935:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574225.793944:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574225.793950:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574225.793954:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574225.793958:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574225.793961:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574225.793965:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574225.793967:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574225.793968:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574225.793970:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61b40 going to sleep for 0 seconds 00000100:00000001:0.0:1788574225.793972:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574225.793974:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574225.793977:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574225.793978:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574225.793980:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574225.793986:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574225.793992:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810235392:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574225.793997:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574225.793999:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574225.794000:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574225.794002:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574225.794005:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574225.794010:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574225.794011:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574225.794016:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574225.794017:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574225.794019:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574225.794022:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574225.794024:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574225.794025:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788574225.794027:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574225.794029:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788574225.794031:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794035:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574225.794038:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794043:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574225.794045:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794056:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574225.794057:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574225.794059:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794063:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574225.794064:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574225.794066:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574225.794068:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574225.794072:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574225.794074:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574225.794076:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794079:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574225.794080:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574225.794081:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574225.794082:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788574225.794083:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574225.794084:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574225.794086:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574225.794088:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574225.794093:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574225.794094:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574225.794098:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574225.794103:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574225.794104:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574225.794106:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574225.794108:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574225.794110:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574225.794111:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574225.794113:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574225.794114:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574225.794116:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168bd8 from hash 00010000:00000001:0.0:1788574225.794117:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574225.794118:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574225.794119:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574225.794119:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574225.794121:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574225.794122:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574225.794124:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574225.794125:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574225.794126:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574225.794127:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574225.794129:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574225.794132:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574225.794133:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574225.794134:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574225.794135:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574225.794136:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574225.794137:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574225.794137:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574225.794139:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bd8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574225.794143:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000001:0.0:1788574225.794150:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574225.794151:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574225.794153:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574225.794158:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810235392/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574225.794163:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574225.794166:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574225.794167:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574225.794169:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574225.794171:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788574225.794174:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574225.794176:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574225.794177:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574225.794180:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574225.794181:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574225.794183:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574225.794185:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574225.794189:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574229.633748:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309368607, at 4309368640 00000100:00000010:0.0:1788574229.633769:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215360. 00000100:00000040:0.0:1788574234.689739:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309373663, at 4309373696 00000100:00000010:0.0:1788574234.689754:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00000040:0.0:1788574239.809722:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309378782, at 4309378816 00000100:00000010:0.0:1788574239.809745:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000100:00000040:0.0:1788574244.929709:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309383903, at 4309383936 00000100:00000010:0.0:1788574244.929724:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00000040:0.0:1788574251.777725:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309390751, at 4309390784 00000100:00000010:0.0:1788574251.777743:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788574257.217708:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309396190, at 4309396224 00000100:00000010:0.0:1788574257.217738:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215060. 00000800:00000200:0.0:1788574257.409972:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574257.409979:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574257.409988:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574257.409992:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574257.409994:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574262.337766:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309401310, at 4309401344 00000100:00000010:0.0:1788574262.337788:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00000040:0.0:1788574267.457741:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309406430, at 4309406464 00000100:00000010:0.0:1788574267.457763:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00000040:0.0:1788574273.601743:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309412575, at 4309412608 00000100:00000010:0.0:1788574273.601764:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215300. 00000100:00000040:0.0:1788574278.721712:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309417694, at 4309417728 00000100:00000010:0.0:1788574278.721728:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215c60. 00000400:00000100:0.0:1788574283.073887:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788574283.073927:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574283.073937:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574283.073941:0:2391:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788574283.073946:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574283.073950:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574283.073955:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574283.073959:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498300. 00000400:00000200:0.0:1788574283.073966:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574283.073971:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574283.073978:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1bb0 00000400:00000010:0.0:1788574283.073981:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1bb0. 00000100:00000001:0.0:1788574283.073986:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574283.073993:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68e00 x1875440810243456/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074016:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574283.074020:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68e00 x1875440810243456/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074026:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.074027:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574283.074035:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574283.074038:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498f00. 00000400:00000200:0.0:1788574283.074055:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574283.074060:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574283.074064:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c14c8 00000400:00000010:0.0:1788574283.074066:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c14c8. 00000100:00000001:0.0:1788574283.074069:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574283.074076:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69c00 x1875440810243584/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074091:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574283.074100:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810243584/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074129:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.074132:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574283.074140:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574283.074148:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498600. 00000400:00000200:0.0:1788574283.074156:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574283.074162:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574283.074170:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1f68 00000400:00000010:0.0:1788574283.074174:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1f68. 00000100:00000001:0.0:1788574283.074180:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574283.074202:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68000 x1875440810243712/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074221:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574283.074225:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810243712/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074237:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.074241:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574283.074252:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574283.074259:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498e00. 00000400:00000200:0.0:1788574283.074264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574283.074274:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574283.074280:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1ee0 00000400:00000010:0.0:1788574283.074301:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1ee0. 00000100:00000001:0.0:1788574283.074309:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574283.074333:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68a80 x1875440810243840/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074346:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574283.074351:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68a80 x1875440810243840/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074362:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.074367:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574283.074379:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574283.074386:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498c00. 00000400:00000200:0.0:1788574283.074391:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574283.074402:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574283.074408:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1770 00000400:00000010:0.0:1788574283.074412:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1770. 00000100:00000001:0.0:1788574283.074435:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574283.074444:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69500 x1875440810243968/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074463:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574283.074470:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69500 x1875440810243968/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574283.074486:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.074492:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788574283.841734:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309422814, at 4309422848 00000100:00000010:0.0:1788574283.841751:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322153c0. 00000100:00000040:0.0:1788574288.961717:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309427934, at 4309427968 00000100:00000010:0.0:1788574288.961731:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322150c0. 10000000:00000001:0.0:1788574290.241736:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574290.241750:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9fc0. 00000020:00000010:0.0:1788574290.241758:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574290.241764:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788574290.241785:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574290.241793:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574290.241801:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788574290.241809:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574290.241817:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574290.241822:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788574290.241828:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574290.241832:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574290.241839:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b400. 00000040:00000010:0.0:1788574290.241846:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574290.241850:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574290.241855:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788574290.241861:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574290.241864:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574290.241869:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574290.241873:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788574290.241874:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9fc0. 10000000:01000000:0.0:1788574290.241877:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574290.241879:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574290.241880:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574290.241882:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574290.241885:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574290.241887:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574290.241890:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574290.241900:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574290.241904:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574290.241908:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574290.241913:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574290.241915:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574290.241918:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574292.033707:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574292.033714:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574292.033719:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574292.033722:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574292.033723:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574292.033738:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574292.033741:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574292.033742:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574292.033745:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574292.033748:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574292.033750:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574292.033758:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574292.033764:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574292.033770:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574292.033774:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574292.033781:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574292.033790:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b600. 00000100:00000001:0.0:1788574292.033798:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574292.033803:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574292.033806:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574292.033812:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692100. 00010000:00000001:0.0:1788574292.033815:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574292.033817:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788574292.033821:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574292.033823:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168bed to hash 00000020:00000001:0.0:1788574292.033825:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574292.033826:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574292.033829:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574292.033832:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574292.033835:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574292.033838:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574292.033846:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574292.033856:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574292.033860:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574292.033866:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574292.033871:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574292.033872:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611584 : -109714523940032 : ffff9c3718c61b40) 00000100:00000001:0.0:1788574292.033875:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574292.033876:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574292.033880:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574292.033884:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574292.033886:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574292.033887:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574292.033896:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574292.033903:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574292.033906:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574292.033911:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574292.033914:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574292.033918:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574292.033920:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574292.033921:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574292.033923:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61b40 going to sleep for 0 seconds 00000100:00000001:0.0:1788574292.033925:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574292.033926:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574292.033929:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574292.033931:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574292.033932:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574292.033935:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574292.033940:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810245376:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574292.033944:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574292.033949:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574292.033951:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574292.033953:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574292.033956:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574292.033961:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574292.033962:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574292.033966:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574292.033967:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574292.033969:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574292.033971:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574292.033972:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574292.033974:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788574292.033976:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574292.033978:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788574292.033980:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.033985:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574292.033988:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.033991:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574292.033993:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.033997:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574292.033998:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574292.034000:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.034004:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574292.034010:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574292.034021:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574292.034025:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574292.034032:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574292.034034:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574292.034037:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.034044:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574292.034045:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574292.034047:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574292.034049:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788574292.034051:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574292.034052:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574292.034054:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574292.034056:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574292.034061:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574292.034063:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574292.034067:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574292.034069:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574292.034071:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574292.034074:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574292.034076:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574292.034079:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574292.034080:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574292.034082:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574292.034084:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574292.034086:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168bed from hash 00010000:00000001:0.0:1788574292.034096:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574292.034105:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574292.034107:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574292.034108:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574292.034110:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574292.034111:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574292.034114:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574292.034115:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574292.034117:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574292.034119:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574292.034122:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574292.034137:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574292.034139:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574292.034140:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574292.034142:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574292.034143:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574292.034144:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574292.034145:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574292.034148:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168bed lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574292.034156:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000001:0.0:1788574292.034165:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574292.034167:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574292.034169:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574292.034174:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810245376/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574292.034195:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574292.034198:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574292.034200:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574292.034201:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574292.034204:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b600. 02000000:00000001:0.0:1788574292.034207:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574292.034209:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574292.034215:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574292.034218:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574292.034218:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574292.034220:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574292.034222:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574292.034224:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000800:00000001:0.0:1788574295.298002:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574295.298022:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574295.298028:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574295.298031:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574295.873768:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309434847, at 4309434880 00000100:00000010:0.0:1788574295.873790:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215b40. 00000100:00000040:0.0:1788574301.249719:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309440222, at 4309440256 00000100:00000010:0.0:1788574301.249732:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215e40. 00000100:00000040:0.0:1788574306.369729:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309445342, at 4309445376 00000100:00000010:0.0:1788574306.369750:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215c00. 00000100:00000040:0.0:1788574311.489735:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309450463, at 4309450496 00000100:00000010:0.0:1788574311.489752:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322154e0. 10000000:00000001:0.0:1788574312.257783:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574312.257817:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9280. 00000020:00000010:0.0:1788574312.257829:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574312.257837:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574312.257844:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574312.257852:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574312.257861:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788574312.257869:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574312.257877:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788574312.257883:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ae00. 10000000:00000040:0.0:1788574312.257890:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574312.257893:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574312.257900:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574312.257909:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574312.257933:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574312.257939:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788574312.257945:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574312.257950:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788574312.257956:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574312.257960:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574312.257965:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9280. 10000000:01000000:0.0:1788574312.257971:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574312.257976:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574312.257980:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574312.257983:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574312.257986:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574312.257990:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574312.257994:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574312.257999:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574312.258002:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574312.258005:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574312.258011:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574312.258014:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574312.258017:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574314.241775:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574314.241785:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574314.241791:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574314.241794:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574314.241796:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574314.241798:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574314.241801:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574314.241803:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574314.241805:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574314.241809:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574314.241811:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574314.241822:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574314.241828:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574314.241833:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574314.241836:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574314.241843:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574314.241852:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788574314.241874:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574314.241879:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574314.241883:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574314.241890:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692100. 00010000:00000001:0.0:1788574314.241895:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574314.241899:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788574314.241914:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574314.241918:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168bf4 to hash 00000020:00000001:0.0:1788574314.241920:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574314.241922:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574314.241925:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574314.241929:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574314.241933:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574314.241938:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574314.241949:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574314.241961:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574314.241967:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574314.241969:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574314.241976:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574314.241979:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611584 : -109714523940032 : ffff9c3718c61b40) 00000100:00000001:0.0:1788574314.241981:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574314.241983:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788574314.241988:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574314.241993:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574314.241995:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574314.241997:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574314.242008:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574314.242030:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574314.242035:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574314.242042:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574314.242046:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574314.242054:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574314.242057:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574314.242059:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574314.242061:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61b40 going to sleep for 0 seconds 00000100:00000001:0.0:1788574314.242064:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574314.242066:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574314.242070:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574314.242073:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574314.242075:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574314.242079:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574314.242087:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810248704:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574314.242092:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574314.242095:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574314.242097:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574314.242100:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574314.242104:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574314.242111:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574314.242114:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61b40. 00000100:00000001:0.0:1788574314.242119:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574314.242121:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574314.242124:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574314.242132:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574314.242134:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574314.242137:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788574314.242139:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574314.242142:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788574314.242146:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242153:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574314.242157:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242163:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574314.242166:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242171:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574314.242173:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574314.242176:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242183:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574314.242184:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574314.242187:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574314.242191:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574314.242197:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574314.242209:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574314.242212:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242227:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574314.242228:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574314.242230:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574314.242231:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788574314.242234:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574314.242236:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574314.242239:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574314.242242:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574314.242248:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574314.242251:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574314.242257:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574314.242259:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574314.242261:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574314.242277:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574314.242279:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574314.242282:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574314.242285:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574314.242286:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574314.242288:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574314.242291:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168bf4 from hash 00010000:00000001:0.0:1788574314.242294:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574314.242295:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574314.242297:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574314.242298:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574314.242300:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574314.242302:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574314.242304:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574314.242305:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574314.242307:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574314.242314:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574314.242317:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574314.242324:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574314.242325:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574314.242327:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574314.242328:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574314.242330:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574314.242331:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574314.242332:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574314.242335:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168bf4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574314.242342:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000001:0.0:1788574314.242352:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574314.242361:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574314.242364:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574314.242369:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810248704/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574314.242376:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574314.242379:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574314.242381:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574314.242383:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574314.242386:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788574314.242391:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574314.242393:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574314.242395:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788574314.242399:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574314.242400:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574314.242402:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574314.242405:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574314.242408:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574318.081805:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309457055, at 4309457088 00000100:00000010:0.0:1788574318.081826:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215b40. 00000100:00000040:0.0:1788574323.265738:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309462239, at 4309462272 00000100:00000010:0.0:1788574323.265764:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00080000:0.0:1788574324.545785:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574324.545799:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574324.545802:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574324.545807:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574324.545813:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574324.545822:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14791 00000400:00000200:0.0:1788574324.545834:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574324.545840:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574324.545845:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574324.545848:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.545857:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.545861:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.545863:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574324.545868:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574324.545871:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574324.545874:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574324.545878:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574324.545881:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574324.545883:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574324.545887:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.545889:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.545891:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.545893:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574324.545897:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574324.545900:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574324.545903:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574324.545907:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574324.545911:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574324.545936:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.545944:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574324.545947:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574324.545955:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574324.545960:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574324.545964:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574324.545965:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.545969:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574324.545973:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574324.545979:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574324.545988:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574324.545996:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.546007:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810250112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574324.546018:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574324.546024:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.546029:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810250112/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574324.546057:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574324.546061:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574324.546065:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574324.546067:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574324.546069:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574324.546073:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574324.546077:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14791 00000400:00000200:0.0:1788574324.546082:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574324.546089:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574324.546093:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574324.546095:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546099:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546111:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546124:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574324.546131:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574324.546134:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574324.546138:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574324.546142:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574324.546146:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574324.546152:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574324.546162:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546168:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546175:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546181:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574324.546187:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574324.546193:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546200:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574324.546204:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574324.546212:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574324.546219:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546223:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574324.546229:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574324.546234:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546237:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574324.546245:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574324.546251:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574324.546258:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574324.546261:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546265:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574324.546268:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574324.546271:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574324.546278:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574324.546285:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.546295:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810250240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574324.546326:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574324.546330:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.546339:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810250240/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574324.546356:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574324.546374:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574324.546379:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574324.546381:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574324.546383:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574324.546389:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574324.546397:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14791 00000400:00000200:0.0:1788574324.546423:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574324.546431:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574324.546436:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574324.546442:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546447:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546452:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546458:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574324.546465:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574324.546469:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574324.546473:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574324.546478:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574324.546483:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574324.546488:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574324.546497:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546501:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546506:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546510:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574324.546521:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574324.546529:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546535:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574324.546540:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574324.546545:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574324.546549:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546555:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574324.546559:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574324.546567:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546571:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574324.546576:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574324.546583:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574324.546589:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574324.546592:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546598:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574324.546600:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574324.546602:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574324.546608:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574324.546614:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.546625:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810250368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574324.546694:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574324.546700:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.546710:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810250368/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574324.546726:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574324.546732:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574324.546739:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574324.546744:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574324.546750:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574324.546758:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574324.546771:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14791 00000400:00000200:0.0:1788574324.546782:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574324.546789:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574324.546796:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574324.546803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546807:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546812:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546816:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574324.546821:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574324.546823:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574324.546824:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574324.546827:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574324.546832:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574324.546838:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574324.546844:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546846:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.546848:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546852:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574324.546857:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574324.546864:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574324.546871:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574324.546875:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574324.546881:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574324.546888:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546894:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574324.546899:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574324.546906:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546911:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574324.546918:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574324.546924:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574324.546931:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574324.546942:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.546947:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574324.546951:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574324.546956:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574324.546963:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574324.546968:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.546974:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810250496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574324.546988:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574324.546994:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.547004:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810250496/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574324.547022:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574324.547027:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574324.547035:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574324.547040:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574324.547044:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574324.547050:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574324.547056:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14791 00000400:00000200:0.0:1788574324.547063:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574324.547070:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574324.547075:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574324.547082:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.547088:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.547094:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.547097:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574324.547100:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574324.547103:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574324.547105:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574324.547109:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574324.547114:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574324.547126:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574324.547132:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.547136:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574324.547141:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574324.547147:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574324.547152:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574324.547160:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574324.547168:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574324.547174:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574324.547180:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574324.547187:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.547194:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574324.547198:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574324.547205:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.547207:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574324.547211:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574324.547217:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574324.547223:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574324.547226:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574324.547231:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574324.547234:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574324.547237:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574324.547246:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574324.547253:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.547265:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810250624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574324.547282:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574324.547286:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.547295:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810250624/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574324.547312:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.547326:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14801) 00000100:00000040:0.0:1788574328.385747:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309467358, at 4309467392 00000100:00000010:0.0:1788574328.385763:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00000040:0.0:1788574333.505718:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309472478, at 4309472512 00000100:00000010:0.0:1788574333.505726:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322152a0. 00000100:00000040:0.0:1788574339.329849:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309478303, at 4309478336 00000100:00000010:0.0:1788574339.329854:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00080000:0.0:1788574340.929816:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574340.929829:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574340.929832:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574340.929836:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574340.929842:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574340.929852:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14807 00000400:00000200:0.0:1788574340.929866:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574340.929875:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574340.929880:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574340.929885:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.929889:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.929893:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.929896:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574340.929901:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574340.929904:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574340.929908:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574340.929913:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574340.929916:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574340.929919:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574340.929925:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.929927:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.929930:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.929932:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574340.929957:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574340.929963:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574340.929966:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574340.929970:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574340.929975:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574340.929978:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.929980:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574340.929983:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574340.929990:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574340.929993:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574340.929997:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574340.929999:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930002:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574340.930004:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574340.930008:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574340.930016:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574340.930024:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.930034:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810252800/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574340.930044:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574340.930049:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.930053:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810252800/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574340.930075:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574340.930077:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574340.930081:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574340.930084:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574340.930089:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574340.930094:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574340.930107:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14807 00000400:00000200:0.0:1788574340.930116:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574340.930135:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574340.930141:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574340.930154:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930158:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930162:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930165:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574340.930167:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574340.930168:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574340.930169:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574340.930171:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574340.930178:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574340.930190:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574340.930207:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930213:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930217:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930222:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574340.930230:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574340.930239:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930246:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574340.930249:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574340.930260:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574340.930267:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930270:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574340.930278:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574340.930284:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930287:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574340.930298:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574340.930307:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574340.930319:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574340.930358:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930362:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574340.930372:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574340.930378:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574340.930383:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574340.930389:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.930399:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810252928/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574340.930416:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574340.930422:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.930431:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810252928/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574340.930454:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574340.930464:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574340.930474:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574340.930476:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574340.930478:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574340.930482:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574340.930486:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14807 00000400:00000200:0.0:1788574340.930492:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574340.930498:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574340.930500:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574340.930507:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930510:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930513:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930527:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574340.930530:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574340.930532:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574340.930535:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574340.930538:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574340.930541:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574340.930549:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574340.930571:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930576:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930582:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930584:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574340.930592:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574340.930599:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930604:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574340.930611:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574340.930615:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574340.930624:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930626:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574340.930629:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574340.930632:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930675:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574340.930682:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574340.930687:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574340.930697:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574340.930702:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.930707:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574340.930714:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574340.930721:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574340.930728:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574340.930737:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.930752:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810253056/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574340.930760:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574340.930769:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.930774:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810253056/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574340.930795:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574340.930805:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574340.930821:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574340.930824:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574340.930825:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574340.930832:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574340.930839:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14807 00000400:00000200:0.0:1788574340.930852:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574340.930862:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574340.930867:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574340.930874:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930879:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930882:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930886:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574340.930896:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574340.930906:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574340.930914:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574340.930924:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574340.930933:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574340.930948:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574340.930961:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930971:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.930982:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.930991:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574340.931002:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574340.931015:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931021:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574340.931031:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574340.931039:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574340.931052:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931064:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574340.931077:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574340.931099:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931105:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574340.931114:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574340.931122:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574340.931133:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574340.931138:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931143:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574340.931147:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574340.931154:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574340.931162:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574340.931167:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.931177:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810253184/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574340.931208:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574340.931214:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.931226:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810253184/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574340.931239:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574340.931242:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574340.931248:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574340.931251:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574340.931254:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574340.931259:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574340.931268:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14807 00000400:00000200:0.0:1788574340.931285:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574340.931300:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574340.931308:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574340.931324:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931336:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.931345:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931361:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574340.931371:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574340.931373:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574340.931376:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574340.931383:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574340.931395:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574340.931405:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574340.931414:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931417:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574340.931422:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931428:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574340.931437:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574340.931448:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574340.931455:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574340.931461:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574340.931467:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574340.931486:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931501:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574340.931506:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574340.931511:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931513:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574340.931529:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574340.931534:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574340.931543:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574340.931547:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574340.931549:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574340.931553:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574340.931561:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574340.931573:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574340.931583:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.931591:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810253312/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574340.931689:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574340.931695:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.931709:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810253312/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574340.931725:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574340.931734:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14817) 00000100:00000040:0.0:1788574344.769784:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309483743, at 4309483776 00000100:00000010:0.0:1788574344.769804:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00080000:0.0:1788574346.049722:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574346.049731:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574346.049733:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574346.049736:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574346.049740:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574346.049745:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14812 00000400:00000200:0.0:1788574346.049752:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574346.049757:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574346.049760:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574346.049762:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049765:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.049767:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049769:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574346.049771:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574346.049773:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574346.049775:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574346.049783:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574346.049784:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574346.049786:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574346.049788:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049806:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.049807:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049809:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574346.049811:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574346.049813:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049815:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574346.049817:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574346.049821:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574346.049823:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.049824:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574346.049826:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574346.049831:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574346.049833:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574346.049837:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574346.049838:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.049841:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574346.049843:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574346.049846:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574346.049853:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574346.049859:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.049866:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810253440/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574346.049874:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574346.049882:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.049886:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810253440/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574346.049904:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574346.049906:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574346.049909:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574346.049910:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574346.049911:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574346.049913:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574346.049924:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14812 00000400:00000200:0.0:1788574346.049928:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574346.049931:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574346.049933:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574346.049935:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049937:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.049942:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049944:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574346.049947:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574346.049950:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574346.049951:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574346.049954:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574346.049957:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574346.049961:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574346.049968:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049972:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.049976:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049980:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574346.049984:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574346.049988:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574346.049991:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574346.049994:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574346.050001:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574346.050005:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050007:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574346.050012:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574346.050017:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050018:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574346.050022:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574346.050027:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574346.050030:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574346.050036:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050042:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574346.050045:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574346.050047:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574346.050053:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574346.050059:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.050071:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810253568/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574346.050089:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574346.050093:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.050101:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810253568/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574346.050115:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574346.050121:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574346.050130:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574346.050133:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574346.050134:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574346.050138:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574346.050142:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14812 00000400:00000200:0.0:1788574346.050173:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574346.050180:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574346.050182:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574346.050184:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050186:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050187:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050192:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574346.050194:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574346.050195:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574346.050196:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574346.050197:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574346.050202:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574346.050212:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574346.050216:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050219:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050223:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050229:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574346.050236:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574346.050245:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050252:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574346.050256:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574346.050263:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574346.050270:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050276:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574346.050280:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574346.050283:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050287:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574346.050291:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574346.050296:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574346.050300:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574346.050304:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050307:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574346.050309:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574346.050311:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574346.050317:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574346.050332:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.050346:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810253696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574346.050363:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574346.050370:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.050375:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810253696/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574346.050389:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574346.050399:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574346.050403:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574346.050406:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574346.050408:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574346.050413:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574346.050418:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14812 00000400:00000200:0.0:1788574346.050426:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574346.050433:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574346.050436:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574346.050440:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050442:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050446:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050449:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574346.050455:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574346.050456:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574346.050457:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574346.050461:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574346.050465:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574346.050468:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574346.050473:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050474:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050477:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050479:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574346.050481:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574346.050487:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050492:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574346.050496:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574346.050500:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574346.050506:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050510:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574346.050520:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574346.050526:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050531:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574346.050538:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574346.050546:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574346.050554:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574346.050557:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050562:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574346.050578:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574346.050580:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574346.050586:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574346.050594:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.050603:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810253824/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574346.050620:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574346.050625:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.050633:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810253824/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574346.050694:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574346.050701:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574346.050705:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574346.050707:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574346.050711:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574346.050717:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574346.050724:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14812 00000400:00000200:0.0:1788574346.050735:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574346.050746:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574346.050752:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574346.050759:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050762:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050768:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050780:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574346.050785:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574346.050789:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574346.050793:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574346.050797:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574346.050804:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574346.050811:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574346.050821:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050828:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574346.050836:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050841:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574346.050848:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574346.050857:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574346.050864:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574346.050868:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574346.050874:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574346.050881:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050887:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574346.050892:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574346.050899:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050905:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574346.050914:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574346.050921:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574346.050925:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574346.050928:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.050929:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574346.050933:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574346.050938:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574346.050945:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574346.050953:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.050963:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810253952/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574346.051002:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574346.051007:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.051016:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810253952/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574346.051032:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574346.051041:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14822) 00000100:00000040:0.0:1788574349.889772:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309488862, at 4309488896 00000100:00000010:0.0:1788574349.889800:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00000040:0.0:1788574355.009782:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309493983, at 4309494016 00000100:00000010:0.0:1788574355.009804:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00080000:0.0:1788574356.289817:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574356.289829:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574356.289831:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574356.289836:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574356.289841:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574356.289848:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14822 00000400:00000200:0.0:1788574356.289858:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574356.289865:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574356.289868:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574356.289871:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.289873:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.289876:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.289877:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574356.289881:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574356.289882:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574356.289885:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574356.289887:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574356.289889:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574356.289890:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574356.289909:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.289910:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.289912:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.289913:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574356.289915:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574356.289918:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574356.289920:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574356.289922:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574356.289925:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574356.289927:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.289929:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574356.289931:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574356.289936:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574356.289939:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574356.289942:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574356.289944:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.289946:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574356.289949:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574356.289953:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574356.289959:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574356.289965:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.289971:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810254720/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574356.289979:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574356.289983:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.289987:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810254720/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574356.290010:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574356.290013:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574356.290016:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574356.290017:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574356.290022:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574356.290024:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574356.290026:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14822 00000400:00000200:0.0:1788574356.290030:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574356.290032:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574356.290034:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574356.290036:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290038:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290040:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290042:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574356.290044:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574356.290045:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574356.290046:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574356.290047:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574356.290049:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574356.290050:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574356.290056:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290058:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290060:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290062:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574356.290064:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574356.290066:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290067:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574356.290069:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574356.290075:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574356.290081:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290082:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574356.290086:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574356.290093:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290094:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574356.290103:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574356.290110:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574356.290121:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574356.290125:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290136:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574356.290147:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574356.290153:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574356.290167:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574356.290174:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.290184:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810254848/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574356.290196:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574356.290200:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.290213:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810254848/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574356.290235:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574356.290243:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574356.290246:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574356.290248:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574356.290252:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574356.290258:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574356.290264:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14822 00000400:00000200:0.0:1788574356.290274:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574356.290280:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574356.290290:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574356.290304:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290306:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290308:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290309:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574356.290310:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574356.290324:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574356.290331:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574356.290334:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574356.290336:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574356.290338:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574356.290342:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290343:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290344:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290350:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574356.290352:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574356.290360:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290363:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574356.290365:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574356.290369:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574356.290382:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290385:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574356.290387:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574356.290395:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290399:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574356.290402:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574356.290404:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574356.290408:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574356.290411:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290413:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574356.290414:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574356.290418:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574356.290426:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574356.290435:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.290442:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810254976/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574356.290456:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574356.290463:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.290482:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810254976/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574356.290494:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574356.290499:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574356.290510:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574356.290525:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574356.290527:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574356.290537:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574356.290545:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14822 00000400:00000200:0.0:1788574356.290555:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574356.290565:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574356.290573:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574356.290582:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290587:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290594:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290601:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574356.290604:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574356.290607:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574356.290612:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574356.290621:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574356.290627:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574356.290629:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574356.290633:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290715:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290722:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290730:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574356.290733:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574356.290740:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290747:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574356.290757:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574356.290762:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574356.290766:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290771:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574356.290774:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574356.290780:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290789:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574356.290795:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574356.290802:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574356.290807:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574356.290812:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.290817:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574356.290819:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574356.290822:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574356.290833:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574356.290842:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.290856:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810255104/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574356.290874:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574356.290876:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.290880:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810255104/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574356.290888:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574356.290897:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574356.290902:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574356.290905:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574356.290908:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574356.290912:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574356.290915:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14822 00000400:00000200:0.0:1788574356.290925:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574356.290931:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574356.290933:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574356.290944:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290947:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290950:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290952:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574356.290955:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574356.290958:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574356.290961:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574356.290965:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574356.290970:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574356.290973:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574356.290977:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290980:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574356.290983:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574356.290986:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574356.290994:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574356.290999:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574356.291003:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574356.291005:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574356.291010:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574356.291012:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.291015:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574356.291019:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574356.291026:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.291031:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574356.291037:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574356.291043:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574356.291054:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574356.291056:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.291059:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574356.291070:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574356.291079:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574356.291092:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574356.291105:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.291115:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810255232/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574356.291157:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574356.291164:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.291179:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810255232/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574356.291211:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574356.291226:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14832) 00000100:00080000:0.0:1788574357.122469:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574357.122475:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574357.122476:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574357.122479:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574357.122482:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574357.122486:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14827 00000400:00000200:0.0:1788574357.122492:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574357.122497:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574357.122499:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574357.122502:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122504:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122506:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122508:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574357.122510:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574357.122512:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574357.122513:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574357.122515:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574357.122516:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574357.122518:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574357.122520:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122521:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122528:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122530:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574357.122532:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574357.122535:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122537:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574357.122539:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574357.122541:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574357.122543:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122544:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574357.122546:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574357.122549:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574357.122551:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574357.122553:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574357.122555:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122556:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574357.122558:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574357.122560:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574357.122563:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574357.122566:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.122571:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810255488/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574357.122577:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574357.122579:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.122583:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810255488/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574357.122604:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574357.122607:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574357.122610:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574357.122611:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574357.122612:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574357.122614:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574357.122620:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14827 00000400:00000200:0.0:1788574357.122624:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574357.122626:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574357.122627:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574357.122629:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122631:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122633:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122660:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574357.122664:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574357.122668:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574357.122684:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574357.122688:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574357.122691:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574357.122697:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574357.122705:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122709:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122713:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122716:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574357.122721:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574357.122724:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122729:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574357.122733:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574357.122740:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574357.122745:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122750:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574357.122756:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574357.122761:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122767:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574357.122776:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574357.122781:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574357.122786:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574357.122797:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122800:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574357.122802:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574357.122805:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574357.122812:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574357.122815:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.122820:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810255616/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574357.122828:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574357.122830:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.122833:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810255616/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574357.122839:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574357.122845:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574357.122850:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574357.122852:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574357.122855:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574357.122858:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574357.122862:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14827 00000400:00000200:0.0:1788574357.122869:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574357.122874:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574357.122879:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574357.122886:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122890:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122894:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122898:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574357.122900:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574357.122903:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574357.122906:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574357.122908:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574357.122911:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574357.122920:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574357.122927:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122929:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.122933:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122938:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574357.122940:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574357.122944:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574357.122950:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574357.122952:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574357.122955:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574357.122958:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122961:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574357.122964:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574357.122969:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122972:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574357.122977:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574357.122981:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574357.122984:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574357.122987:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.122990:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574357.122992:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574357.122994:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574357.123000:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574357.123004:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.123010:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810255744/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574357.123025:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574357.123028:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.123033:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810255744/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574357.123043:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574357.123051:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574357.123056:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574357.123059:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574357.123061:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574357.123067:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574357.123071:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14827 00000400:00000200:0.0:1788574357.123078:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574357.123082:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574357.123086:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574357.123092:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123096:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.123100:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123103:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574357.123109:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574357.123110:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574357.123113:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574357.123115:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574357.123120:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574357.123121:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574357.123125:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123128:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.123133:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123135:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574357.123139:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574357.123144:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123146:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574357.123147:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574357.123149:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574357.123150:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123152:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574357.123161:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574357.123166:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123167:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574357.123170:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574357.123173:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574357.123177:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574357.123178:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123182:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574357.123185:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574357.123189:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574357.123197:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574357.123204:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.123214:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810255872/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574357.123229:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574357.123235:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.123244:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810255872/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574357.123259:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574357.123263:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574357.123267:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574357.123270:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574357.123272:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574357.123278:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574357.123283:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14827 00000400:00000200:0.0:1788574357.123295:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574357.123297:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574357.123299:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574357.123301:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123305:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.123308:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123314:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574357.123318:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574357.123321:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574357.123324:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574357.123326:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574357.123329:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574357.123333:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574357.123338:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123342:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574357.123349:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123352:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574357.123355:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574357.123362:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574357.123367:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574357.123371:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574357.123375:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574357.123380:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123387:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574357.123389:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574357.123390:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123392:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574357.123395:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574357.123398:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574357.123403:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574357.123405:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.123407:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574357.123410:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574357.123412:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574357.123418:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574357.123423:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.123432:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810256000/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574357.123462:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574357.123467:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.123473:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810256000/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574357.123485:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574357.123490:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14833) 00000100:00000040:0.0:1788574360.961803:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309499935, at 4309499968 00000100:00000010:0.0:1788574360.961826:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788574362.433814:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574362.433825:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574362.433827:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574362.433830:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574362.433834:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574362.433841:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14828 00000400:00000200:0.0:1788574362.433849:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574362.433855:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574362.433858:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574362.433860:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.433863:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.433865:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.433867:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574362.433871:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574362.433872:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574362.433875:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574362.433878:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574362.433880:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574362.433882:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574362.433884:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.433894:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.433896:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.433898:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574362.433900:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574362.433903:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574362.433905:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574362.433908:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574362.433911:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574362.433913:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.433915:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574362.433917:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574362.433922:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574362.433925:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574362.433929:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574362.433930:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.433933:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574362.433936:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574362.433940:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574362.433947:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574362.433954:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.433962:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810256128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574362.433970:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574362.433974:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.433979:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810256128/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574362.433995:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574362.433998:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574362.434001:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574362.434002:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574362.434004:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574362.434006:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574362.434014:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14828 00000400:00000200:0.0:1788574362.434018:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574362.434023:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574362.434026:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574362.434032:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434036:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434038:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434041:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574362.434043:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574362.434045:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574362.434046:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574362.434050:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574362.434054:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574362.434058:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574362.434065:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434068:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434071:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434073:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574362.434075:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574362.434076:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434081:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574362.434086:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574362.434091:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574362.434095:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434100:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574362.434104:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574362.434111:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434115:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574362.434118:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574362.434122:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574362.434127:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574362.434135:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434138:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574362.434142:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574362.434145:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574362.434150:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574362.434155:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.434163:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810256256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574362.434174:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574362.434177:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.434182:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810256256/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574362.434187:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574362.434190:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574362.434198:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574362.434200:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574362.434203:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574362.434206:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574362.434211:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14828 00000400:00000200:0.0:1788574362.434219:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574362.434224:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574362.434227:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574362.434232:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434236:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434241:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434243:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574362.434244:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574362.434246:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574362.434248:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574362.434251:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574362.434253:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574362.434260:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574362.434267:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434270:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434275:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434279:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574362.434283:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574362.434288:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434294:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574362.434298:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574362.434301:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574362.434306:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434310:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574362.434312:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574362.434315:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434316:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574362.434321:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574362.434332:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574362.434336:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574362.434338:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434341:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574362.434343:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574362.434347:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574362.434355:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574362.434364:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.434377:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810256384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574362.434396:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574362.434401:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.434410:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810256384/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574362.434420:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574362.434429:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574362.434435:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574362.434438:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574362.434442:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574362.434447:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574362.434451:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14828 00000400:00000200:0.0:1788574362.434458:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574362.434461:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574362.434463:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574362.434465:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434469:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434473:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434476:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574362.434480:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574362.434481:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574362.434485:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574362.434488:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574362.434494:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574362.434497:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574362.434504:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434508:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434514:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434517:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574362.434524:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574362.434531:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434538:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574362.434543:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574362.434549:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574362.434555:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434559:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574362.434585:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574362.434590:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434594:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574362.434598:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574362.434604:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574362.434610:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574362.434614:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434619:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574362.434623:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574362.434626:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574362.434633:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574362.434670:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.434681:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810256512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574362.434699:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574362.434702:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.434708:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810256512/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574362.434719:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574362.434738:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574362.434742:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574362.434745:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574362.434748:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574362.434754:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574362.434760:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14828 00000400:00000200:0.0:1788574362.434768:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574362.434771:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574362.434773:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574362.434780:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434785:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434791:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434804:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574362.434809:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574362.434812:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574362.434813:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574362.434816:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574362.434819:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574362.434823:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574362.434826:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434831:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574362.434833:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434834:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574362.434839:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574362.434845:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574362.434848:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574362.434851:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574362.434854:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574362.434857:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434862:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574362.434864:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574362.434869:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434870:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574362.434872:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574362.434874:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574362.434877:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574362.434881:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.434885:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574362.434886:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574362.434890:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574362.434895:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574362.434899:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.434908:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810256640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574362.434931:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574362.434934:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.434940:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810256640/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574362.434954:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574362.434958:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14839) 00000100:00000040:0.0:1788574366.273741:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309505246, at 4309505280 00000100:00000010:0.0:1788574366.273791:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00080000:0.0:1788574367.553701:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574367.553710:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574367.553712:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574367.553716:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574367.553720:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574367.553726:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14834 00000400:00000200:0.0:1788574367.553733:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574367.553738:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574367.553741:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574367.553744:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553746:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.553749:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553750:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574367.553754:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574367.553755:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574367.553758:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574367.553760:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574367.553762:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574367.553764:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574367.553766:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553794:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.553796:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553797:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574367.553800:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574367.553803:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553805:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574367.553807:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574367.553810:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574367.553812:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.553814:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574367.553816:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574367.553821:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574367.553824:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574367.553827:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574367.553829:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.553831:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574367.553834:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574367.553837:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574367.553844:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574367.553850:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.553857:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810256768/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574367.553867:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574367.553871:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.553876:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810256768/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574367.553902:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574367.553906:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574367.553910:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574367.553912:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574367.553913:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574367.553916:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574367.553928:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14834 00000400:00000200:0.0:1788574367.553945:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574367.553953:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574367.553959:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574367.553964:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553966:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.553969:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.553974:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574367.553979:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574367.553983:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574367.553988:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574367.553993:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574367.553999:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574367.554005:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574367.554013:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554017:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554023:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554029:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574367.554034:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574367.554039:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554044:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574367.554048:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574367.554057:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574367.554063:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554068:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574367.554075:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574367.554082:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554086:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574367.554093:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574367.554099:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574367.554104:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574367.554115:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554120:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574367.554124:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574367.554129:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574367.554139:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574367.554147:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.554158:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810256896/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574367.554175:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574367.554180:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.554187:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810256896/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574367.554212:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574367.554216:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574367.554224:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574367.554228:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574367.554233:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574367.554239:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574367.554245:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14834 00000400:00000200:0.0:1788574367.554256:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574367.554263:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574367.554267:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574367.554275:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554281:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554287:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554291:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574367.554294:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574367.554297:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574367.554301:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574367.554305:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574367.554312:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574367.554324:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574367.554331:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554337:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554343:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554349:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574367.554354:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574367.554361:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554367:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574367.554372:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574367.554376:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574367.554381:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554386:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574367.554390:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574367.554405:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554411:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574367.554417:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574367.554424:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574367.554429:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574367.554432:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554436:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574367.554439:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574367.554443:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574367.554452:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574367.554459:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.554472:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810257024/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574367.554488:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574367.554494:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.554500:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810257024/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574367.554509:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574367.554519:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574367.554525:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574367.554530:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574367.554534:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574367.554541:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574367.554549:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14834 00000400:00000200:0.0:1788574367.554559:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574367.554569:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574367.554575:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574367.554581:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554585:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554590:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554595:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574367.554601:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574367.554604:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574367.554606:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574367.554609:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574367.554614:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574367.554619:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574367.554626:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554629:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554632:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554670:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574367.554678:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574367.554686:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554693:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574367.554698:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574367.554705:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574367.554712:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554717:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574367.554730:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574367.554738:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554744:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574367.554749:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574367.554754:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574367.554757:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574367.554760:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554765:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574367.554768:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574367.554771:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574367.554790:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574367.554796:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.554802:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810257152/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574367.554811:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574367.554814:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.554818:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810257152/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574367.554825:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574367.554827:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574367.554831:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574367.554832:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574367.554834:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574367.554837:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574367.554842:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14834 00000400:00000200:0.0:1788574367.554884:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574367.554890:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574367.554892:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574367.554897:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554901:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554905:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554915:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574367.554920:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574367.554923:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574367.554926:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574367.554928:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574367.554932:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574367.554935:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574367.554942:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554945:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574367.554948:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554951:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574367.554953:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574367.554958:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574367.554962:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574367.554964:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574367.554968:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574367.554972:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554976:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574367.554978:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574367.554982:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.554984:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574367.554990:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574367.554994:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574367.554996:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574367.554999:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.555002:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574367.555004:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574367.555007:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574367.555013:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574367.555020:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.555029:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810257280/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574367.555061:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574367.555066:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.555076:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810257280/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574367.555096:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574367.555104:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14844) 00000100:00000040:0.0:1788574371.393742:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309510366, at 4309510400 00000100:00000010:0.0:1788574371.393757:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788574372.673769:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574372.673781:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574372.673785:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574372.673789:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574372.673793:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574372.673801:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14839 00000400:00000200:0.0:1788574372.673810:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574372.673817:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574372.673820:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574372.673824:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.673841:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.673844:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.673846:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574372.673851:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574372.673854:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574372.673856:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574372.673860:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574372.673863:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574372.673865:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574372.673868:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.673888:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.673890:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.673892:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574372.673896:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574372.673900:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574372.673903:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574372.673906:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574372.673910:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574372.673913:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.673916:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574372.673918:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574372.673925:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574372.673928:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574372.673933:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574372.673935:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.673938:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574372.673941:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574372.673946:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574372.673954:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574372.673961:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.673971:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810257408/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574372.673983:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574372.673987:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.673992:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810257408/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574372.674016:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574372.674019:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574372.674022:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574372.674024:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574372.674026:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574372.674029:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574372.674037:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14839 00000400:00000200:0.0:1788574372.674042:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574372.674047:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574372.674050:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574372.674057:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674063:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674070:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674077:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574372.674084:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574372.674086:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574372.674089:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574372.674091:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574372.674093:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574372.674098:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574372.674106:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674113:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674118:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674123:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574372.674128:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574372.674137:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674142:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574372.674147:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574372.674156:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574372.674160:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674167:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574372.674174:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574372.674181:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674185:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574372.674193:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574372.674198:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574372.674203:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574372.674213:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674219:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574372.674223:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574372.674226:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574372.674233:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574372.674239:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.674250:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810257536/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574372.674265:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574372.674270:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.674278:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810257536/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574372.674298:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574372.674304:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574372.674311:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574372.674314:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574372.674318:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574372.674322:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574372.674331:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14839 00000400:00000200:0.0:1788574372.674368:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574372.674377:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574372.674383:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574372.674389:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674394:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674401:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674406:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574372.674411:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574372.674415:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574372.674419:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574372.674423:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574372.674429:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574372.674441:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574372.674447:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674453:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674459:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674464:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574372.674468:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574372.674472:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674475:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574372.674477:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574372.674480:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574372.674485:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674489:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574372.674493:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574372.674498:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674502:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574372.674508:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574372.674514:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574372.674521:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574372.674523:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674528:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574372.674531:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574372.674535:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574372.674542:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574372.674549:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.674559:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810257664/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574372.674577:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574372.674582:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.674589:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810257664/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574372.674603:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574372.674617:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574372.674627:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574372.674632:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574372.674668:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574372.674675:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574372.674681:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14839 00000400:00000200:0.0:1788574372.674690:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574372.674699:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574372.674704:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574372.674709:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674714:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674719:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674724:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574372.674729:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574372.674733:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574372.674737:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574372.674744:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574372.674752:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574372.674758:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574372.674767:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674773:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.674780:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674785:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574372.674791:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574372.674797:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574372.674802:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574372.674806:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574372.674811:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574372.674818:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674824:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574372.674848:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574372.674854:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674859:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574372.674865:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574372.674871:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574372.674878:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574372.674883:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.674887:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574372.674892:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574372.674898:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574372.674907:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574372.674914:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.674924:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810257792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574372.674933:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574372.674939:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.674948:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810257792/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574372.674981:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574372.674988:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574372.674996:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574372.674998:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574372.675001:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574372.675008:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574372.675015:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14839 00000400:00000200:0.0:1788574372.675026:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574372.675035:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574372.675039:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574372.675046:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.675052:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.675056:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.675068:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574372.675074:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574372.675079:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574372.675084:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574372.675088:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574372.675092:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574372.675097:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574372.675104:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.675107:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574372.675111:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574372.675115:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574372.675120:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574372.675126:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574372.675131:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574372.675135:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574372.675140:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574372.675148:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.675153:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574372.675157:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574372.675164:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.675169:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574372.675176:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574372.675179:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574372.675182:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574372.675184:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.675189:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574372.675192:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574372.675197:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574372.675205:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574372.675213:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.675223:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810257920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574372.675257:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574372.675263:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.675273:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810257920/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574372.675293:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574372.675302:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14849) 00000100:00000040:0.0:1788574376.513739:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309515486, at 4309515520 00000100:00000010:0.0:1788574376.513759:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574377.153824:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574377.153828:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574377.153832:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574378.562405:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574378.562414:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574378.562417:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574378.562422:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574378.562426:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574378.562431:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14849 00000400:00000200:0.0:1788574378.562440:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574378.562448:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574378.562452:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574378.562456:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562459:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.562462:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562465:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574378.562468:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574378.562470:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574378.562473:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574378.562475:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574378.562478:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574378.562488:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574378.562492:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562494:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.562496:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562499:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574378.562502:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574378.562506:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562510:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574378.562513:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574378.562516:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574378.562520:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.562523:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574378.562525:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574378.562530:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574378.562534:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574378.562537:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574378.562539:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.562541:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574378.562544:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574378.562546:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574378.562552:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574378.562557:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.562564:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810258816/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574378.562574:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574378.562577:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.562582:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810258816/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574378.562611:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574378.562615:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574378.562619:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574378.562627:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574378.562629:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574378.562633:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574378.562677:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14849 00000400:00000200:0.0:1788574378.562689:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574378.562699:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574378.562704:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574378.562711:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562717:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.562722:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562725:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574378.562728:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574378.562729:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574378.562733:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574378.562738:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574378.562745:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574378.562751:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574378.562758:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562762:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.562764:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562767:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574378.562772:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574378.562777:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574378.562785:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574378.562790:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574378.562797:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574378.562802:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.562807:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574378.562812:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574378.562818:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.562828:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574378.562834:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574378.562840:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574378.562844:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574378.562847:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.562853:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574378.562865:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574378.562870:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574378.562878:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574378.562900:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.562911:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810258944/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574378.562928:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574378.562933:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.562942:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810258944/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574378.562961:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574378.562966:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574378.562974:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574378.562977:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574378.562982:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574378.562989:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574378.562995:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14849 00000400:00000200:0.0:1788574378.563036:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574378.563046:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574378.563050:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574378.563057:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563062:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563068:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563073:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574378.563080:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574378.563093:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574378.563098:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574378.563103:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574378.563109:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574378.563116:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574378.563123:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563128:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563134:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563138:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574378.563142:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574378.563150:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563155:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574378.563160:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574378.563166:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574378.563172:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563178:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574378.563182:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574378.563187:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563193:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574378.563198:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574378.563205:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574378.563213:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574378.563217:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563222:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574378.563226:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574378.563228:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574378.563236:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574378.563242:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.563252:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810259072/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574378.563273:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574378.563276:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.563293:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810259072/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574378.563327:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574378.563334:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574378.563343:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574378.563349:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574378.563353:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574378.563362:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574378.563370:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14849 00000400:00000200:0.0:1788574378.563381:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574378.563392:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574378.563399:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574378.563409:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563416:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563423:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563431:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574378.563439:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574378.563443:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574378.563448:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574378.563453:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574378.563458:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574378.563462:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574378.563468:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563473:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563475:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563477:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574378.563481:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574378.563485:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563491:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574378.563502:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574378.563508:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574378.563514:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563519:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574378.563524:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574378.563529:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563533:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574378.563541:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574378.563547:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574378.563554:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574378.563558:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563562:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574378.563567:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574378.563572:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574378.563579:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574378.563586:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.563600:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810259200/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574378.563619:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574378.563622:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.563633:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810259200/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574378.563674:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574378.563681:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574378.563688:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574378.563691:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574378.563694:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574378.563702:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574378.563711:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14849 00000400:00000200:0.0:1788574378.563721:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574378.563730:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574378.563737:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574378.563753:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563760:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563768:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563776:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574378.563784:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574378.563789:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574378.563794:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574378.563800:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574378.563806:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574378.563813:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574378.563825:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563832:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574378.563840:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563847:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574378.563853:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574378.563861:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574378.563865:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574378.563871:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574378.563875:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574378.563882:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563903:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574378.563906:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574378.563912:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563918:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574378.563927:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574378.563934:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574378.563943:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574378.563948:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574378.563954:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574378.563959:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574378.563963:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574378.563971:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574378.563986:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.563999:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810259328/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574378.564023:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574378.564028:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.564038:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810259328/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574378.564065:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574378.564074:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14855) 00000020:00000001:0.0:1788574382.273888:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574382.273893:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574382.273897:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574382.401757:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309521375, at 4309521408 00000100:00000010:0.0:1788574382.401776:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00080000:0.0:1788574383.937778:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574383.937786:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574383.937788:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574383.937791:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574383.937796:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574383.937802:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14850 00000400:00000200:0.0:1788574383.937810:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574383.937816:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574383.937820:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574383.937822:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.937825:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.937827:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.937830:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574383.937833:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574383.937834:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574383.937836:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574383.937853:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574383.937855:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574383.937857:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574383.937859:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.937861:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.937863:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.937864:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574383.937867:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574383.937869:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574383.937871:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574383.937873:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574383.937876:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574383.937878:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.937880:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574383.937882:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574383.937887:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574383.937889:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574383.937892:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574383.937894:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.937896:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574383.937899:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574383.937903:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574383.937910:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574383.937917:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.937924:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810259456/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574383.937932:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574383.937936:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.937940:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810259456/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574383.937971:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574383.937974:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574383.937983:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574383.937986:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574383.937989:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574383.937992:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574383.937997:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14850 00000400:00000200:0.0:1788574383.938003:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574383.938011:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574383.938018:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574383.938024:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938027:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938033:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938039:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574383.938047:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574383.938050:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574383.938055:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574383.938060:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574383.938067:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574383.938072:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574383.938080:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938084:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938089:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938092:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574383.938097:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574383.938102:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938108:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574383.938112:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574383.938119:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574383.938124:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938128:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574383.938133:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574383.938147:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938152:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574383.938158:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574383.938164:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574383.938168:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574383.938170:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938174:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574383.938178:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574383.938182:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574383.938188:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574383.938196:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.938207:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810259584/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574383.938224:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574383.938229:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.938238:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810259584/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574383.938264:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574383.938271:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574383.938277:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574383.938281:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574383.938285:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574383.938293:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574383.938297:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14850 00000400:00000200:0.0:1788574383.938320:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574383.938330:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574383.938335:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574383.938343:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938351:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938355:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938366:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574383.938372:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574383.938376:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574383.938382:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574383.938387:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574383.938392:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574383.938395:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574383.938402:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938408:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938415:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938418:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574383.938423:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574383.938431:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938438:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574383.938443:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574383.938448:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574383.938454:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938457:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574383.938461:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574383.938466:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938472:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574383.938480:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574383.938485:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574383.938490:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574383.938494:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938497:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574383.938500:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574383.938505:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574383.938513:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574383.938522:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.938533:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810259712/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574383.938564:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574383.938570:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.938579:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810259712/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574383.938597:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574383.938604:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574383.938611:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574383.938616:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574383.938620:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574383.938626:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574383.938659:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14850 00000400:00000200:0.0:1788574383.938667:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574383.938675:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574383.938681:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574383.938691:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938694:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938699:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938704:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574383.938709:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574383.938712:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574383.938716:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574383.938721:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574383.938727:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574383.938733:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574383.938742:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938747:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.938753:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938759:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574383.938772:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574383.938781:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574383.938786:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574383.938791:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574383.938796:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574383.938803:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938809:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574383.938813:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574383.938819:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938823:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574383.938830:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574383.938837:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574383.938842:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574383.938845:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.938848:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574383.938852:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574383.938854:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574383.938858:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574383.938863:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.938871:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810259840/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574383.938898:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574383.938903:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.938914:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810259840/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574383.938932:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574383.938938:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574383.938961:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574383.938963:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574383.938966:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574383.938971:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574383.938984:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14850 00000400:00000200:0.0:1788574383.938992:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574383.939001:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574383.939007:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574383.939015:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.939019:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.939025:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.939031:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574383.939037:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574383.939041:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574383.939046:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574383.939051:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574383.939054:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574383.939059:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574383.939065:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.939070:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574383.939075:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574383.939080:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574383.939084:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574383.939091:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574383.939100:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574383.939105:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574383.939112:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574383.939120:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.939126:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574383.939130:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574383.939136:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.939140:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574383.939146:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574383.939151:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574383.939158:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574383.939167:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574383.939171:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574383.939173:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574383.939176:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574383.939184:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574383.939191:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.939202:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810259968/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574383.939217:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574383.939221:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.939232:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810259968/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574383.939250:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574383.939257:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14860) 00000020:00000001:0.0:1788574387.393848:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574387.393852:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574387.393855:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574387.777723:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309526750, at 4309526784 00000100:00000010:0.0:1788574387.777744:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788574392.513823:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574392.513827:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574392.513830:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574392.897749:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309531870, at 4309531904 00000100:00000010:0.0:1788574392.897765:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788574397.633823:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574397.633829:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574397.633833:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574398.017737:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309536990, at 4309537024 00000100:00000010:0.0:1788574398.017754:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 10000000:00000001:0.0:1788574398.785775:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574398.785796:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9a80. 00000020:00000010:0.0:1788574398.785807:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574398.785814:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574398.785835:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574398.785843:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574398.785848:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788574398.785855:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574398.785862:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574398.785868:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788574398.785874:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574398.785877:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574398.785884:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788574398.785894:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574398.785897:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574398.785902:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788574398.785905:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574398.785909:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574398.785912:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574398.785915:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b600. 10000000:00000010:0.0:1788574398.785918:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9a80. 10000000:01000000:0.0:1788574398.785922:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574398.785926:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574398.792776:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574398.792780:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574398.792781:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574398.792784:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574398.792786:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574398.792790:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574398.792792:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574398.792795:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574398.792799:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574398.792801:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574398.792805:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788574402.753815:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574402.753819:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574402.753822:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574403.777840:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574403.777847:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574403.777871:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574404.289743:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309543263, at 4309543296 00000100:00000010:0.0:1788574404.289766:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00080000:0.0:1788574405.953810:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574405.953826:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574405.953829:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574405.953834:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574405.953839:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574405.953849:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14872 00000400:00000200:0.0:1788574405.953861:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574405.953869:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574405.953873:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574405.953877:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.953880:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.953883:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.953886:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574405.953891:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574405.953894:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574405.953897:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574405.953902:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574405.953905:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574405.953908:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574405.953911:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.953913:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.953916:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.953917:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574405.953921:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574405.953931:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574405.953934:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574405.953937:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574405.953966:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574405.953970:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.953973:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574405.953976:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574405.953984:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574405.953989:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574405.953994:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574405.953995:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.953999:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574405.954002:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574405.954008:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574405.954017:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574405.954026:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.954038:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810262784/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574405.954051:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574405.954057:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.954063:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810262784/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574405.954093:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574405.954098:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574405.954101:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574405.954103:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574405.954105:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574405.954108:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574405.954111:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14872 00000400:00000200:0.0:1788574405.954117:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574405.954121:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574405.954124:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574405.954131:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954134:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954143:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954145:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574405.954149:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574405.954152:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574405.954156:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574405.954161:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574405.954184:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574405.954191:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574405.954200:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954208:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954218:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954228:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574405.954236:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574405.954245:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954250:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574405.954255:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574405.954267:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574405.954275:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954283:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574405.954291:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574405.954298:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954303:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574405.954313:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574405.954318:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574405.954322:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574405.954326:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954336:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574405.954337:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574405.954341:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574405.954349:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574405.954357:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.954370:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810262912/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574405.954390:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574405.954397:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.954408:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810262912/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574405.954441:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574405.954447:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574405.954457:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574405.954462:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574405.954466:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574405.954477:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574405.954485:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14872 00000400:00000200:0.0:1788574405.954495:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574405.954500:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574405.954503:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574405.954506:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954512:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954517:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954519:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574405.954522:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574405.954523:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574405.954525:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574405.954528:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574405.954530:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574405.954532:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574405.954543:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954547:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954549:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954553:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574405.954570:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574405.954579:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954584:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574405.954590:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574405.954599:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574405.954605:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954614:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574405.954619:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574405.954626:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954631:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574405.954692:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574405.954700:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574405.954706:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574405.954710:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.954715:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574405.954718:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574405.954723:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574405.954732:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574405.954741:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.954757:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810263040/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574405.954776:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574405.954784:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.954796:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810263040/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574405.954829:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574405.954840:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574405.954850:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574405.954856:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574405.954859:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574405.954865:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574405.954878:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14872 00000400:00000200:0.0:1788574405.954888:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574405.954899:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574405.954906:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574405.954916:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954924:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954932:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954937:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574405.954943:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574405.954947:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574405.954952:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574405.954957:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574405.954963:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574405.954966:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574405.954973:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954977:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.954981:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.954987:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574405.954991:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574405.955000:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955006:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574405.955011:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574405.955018:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574405.955030:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955039:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574405.955045:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574405.955053:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955060:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574405.955069:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574405.955075:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574405.955080:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574405.955093:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955100:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574405.955105:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574405.955110:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574405.955119:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574405.955128:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.955146:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810263168/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574405.955197:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574405.955202:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.955213:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810263168/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574405.955256:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574405.955263:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574405.955274:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574405.955279:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574405.955283:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574405.955293:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574405.955302:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14872 00000400:00000200:0.0:1788574405.955311:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574405.955317:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574405.955323:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574405.955333:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955340:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.955348:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955357:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574405.955365:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574405.955369:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574405.955371:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574405.955374:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574405.955377:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574405.955391:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574405.955404:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955412:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574405.955420:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955431:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574405.955439:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574405.955444:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574405.955450:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574405.955456:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574405.955463:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574405.955471:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955480:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574405.955484:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574405.955491:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955497:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574405.955506:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574405.955513:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574405.955520:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574405.955524:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.955528:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574405.955533:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574405.955538:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574405.955547:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574405.955556:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.955570:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810263296/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574405.955585:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574405.955592:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.955605:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810263296/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574405.955622:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574405.955663:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14882) 00000020:00000001:0.0:1788574407.873845:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574407.873849:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574407.873852:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574408.897936:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574408.897954:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574408.897958:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574409.793747:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309548767, at 4309548800 00000100:00000010:0.0:1788574409.793768:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000800:00000200:0.0:1788574411.009950:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574411.009957:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574411.009970:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574411.009975:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574411.009979:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788574411.073727:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574411.073737:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574411.073740:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574411.073743:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574411.073747:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574411.073754:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14877 00000400:00000200:0.0:1788574411.073762:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574411.073767:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574411.073770:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574411.073772:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073775:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.073777:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073779:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574411.073782:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574411.073783:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574411.073786:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574411.073788:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574411.073790:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574411.073792:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574411.073810:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073812:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.073813:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073815:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574411.073817:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574411.073819:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073821:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574411.073823:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574411.073826:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574411.073828:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.073830:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574411.073832:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574411.073837:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574411.073839:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574411.073842:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574411.073844:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.073846:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574411.073848:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574411.073852:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574411.073859:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574411.073865:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.073872:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810263424/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574411.073879:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574411.073882:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.073886:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810263424/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574411.073908:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574411.073910:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574411.073913:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574411.073914:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574411.073919:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574411.073921:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574411.073923:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14877 00000400:00000200:0.0:1788574411.073927:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574411.073929:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574411.073931:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574411.073933:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073935:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.073937:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073945:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574411.073947:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574411.073948:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574411.073949:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574411.073952:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574411.073955:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574411.073958:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574411.073962:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073965:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.073969:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073971:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574411.073974:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574411.073978:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574411.073983:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574411.073987:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574411.073993:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574411.073996:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.073998:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574411.074003:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574411.074007:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074011:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574411.074022:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574411.074025:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574411.074029:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574411.074033:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074035:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574411.074037:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574411.074039:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574411.074042:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574411.074048:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.074054:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810263552/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574411.074062:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574411.074066:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.074071:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810263552/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574411.074083:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574411.074087:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574411.074092:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574411.074093:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574411.074094:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574411.074097:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574411.074101:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14877 00000400:00000200:0.0:1788574411.074105:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574411.074108:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574411.074110:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574411.074116:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074121:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074124:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074127:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574411.074129:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574411.074135:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574411.074138:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574411.074140:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574411.074142:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574411.074143:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574411.074146:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074148:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074151:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074154:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574411.074159:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574411.074163:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074167:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574411.074170:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574411.074174:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574411.074178:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074181:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574411.074182:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574411.074184:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074188:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574411.074192:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574411.074195:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574411.074199:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574411.074201:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074204:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574411.074205:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574411.074206:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574411.074214:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574411.074217:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.074247:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810263680/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574411.074257:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574411.074261:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.074270:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810263680/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574411.074280:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574411.074283:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574411.074289:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574411.074291:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574411.074293:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574411.074297:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574411.074301:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14877 00000400:00000200:0.0:1788574411.074323:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574411.074329:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574411.074333:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574411.074335:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074337:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074341:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074344:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574411.074346:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574411.074347:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574411.074349:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574411.074352:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574411.074356:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574411.074359:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574411.074365:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074369:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074372:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074373:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574411.074375:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574411.074377:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074378:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574411.074382:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574411.074385:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574411.074387:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074389:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574411.074392:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574411.074395:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074401:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574411.074406:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574411.074417:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574411.074420:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574411.074421:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074422:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574411.074424:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574411.074428:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574411.074432:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574411.074436:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.074443:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810263808/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574411.074453:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574411.074458:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.074463:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810263808/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574411.074474:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574411.074477:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574411.074481:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574411.074483:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574411.074485:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574411.074489:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574411.074494:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14877 00000400:00000200:0.0:1788574411.074498:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574411.074501:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574411.074505:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574411.074516:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074519:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074523:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074525:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574411.074528:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574411.074530:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574411.074533:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574411.074535:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574411.074539:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574411.074543:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574411.074551:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074554:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574411.074558:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074563:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574411.074565:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574411.074570:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574411.074574:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574411.074577:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574411.074580:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574411.074583:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074586:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574411.074589:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574411.074593:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074598:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574411.074602:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574411.074606:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574411.074609:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574411.074615:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574411.074619:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574411.074620:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574411.074621:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574411.074625:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574411.074658:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.074664:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810263936/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574411.074671:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574411.074674:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.074679:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810263936/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574411.074696:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.074703:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14887) 00000020:00000001:0.0:1788574412.993836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574412.993841:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574412.993845:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574414.017814:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574414.017819:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574414.017823:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574414.913760:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309553886, at 4309553920 00000100:00000010:0.0:1788574414.913774:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788574418.113871:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574418.113875:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574418.113878:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574419.137792:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574419.137795:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574419.137798:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574420.033713:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309559006, at 4309559040 00000100:00000010:0.0:1788574420.033728:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788574423.233840:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574423.233844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574423.233848:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574424.257865:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574424.257869:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574424.257872:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574425.793718:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309564767, at 4309564800 00000100:00000010:0.0:1788574425.793734:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788574427.457796:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574427.457807:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574427.457810:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574427.457814:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574427.457819:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574427.457826:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14893 00000400:00000200:0.0:1788574427.457835:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574427.457842:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574427.457846:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574427.457850:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.457854:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.457856:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.457859:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574427.457863:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574427.457866:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574427.457869:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574427.457872:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574427.457875:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574427.457878:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574427.457882:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.457884:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.457886:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.457888:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574427.457892:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574427.457896:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574427.457899:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574427.457903:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574427.457906:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574427.457910:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.457912:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574427.457933:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574427.457940:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574427.457944:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574427.457948:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574427.457950:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.457954:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574427.457957:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574427.457962:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574427.457970:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574427.457977:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.457987:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810265600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574427.457999:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574427.458003:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.458009:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810265600/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574427.458034:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574427.458038:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574427.458042:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574427.458044:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574427.458046:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574427.458048:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574427.458052:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14893 00000400:00000200:0.0:1788574427.458057:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574427.458061:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574427.458065:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574427.458068:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458072:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458077:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458081:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574427.458086:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574427.458094:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574427.458099:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574427.458103:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574427.458118:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574427.458124:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574427.458132:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458138:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458144:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458149:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574427.458155:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574427.458160:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458166:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574427.458170:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574427.458178:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574427.458183:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458189:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574427.458196:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574427.458201:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458205:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574427.458211:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574427.458218:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574427.458225:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574427.458228:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458233:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574427.458238:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574427.458243:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574427.458250:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574427.458257:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.458267:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810265728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574427.458281:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574427.458285:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.458300:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810265728/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574427.458324:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574427.458330:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574427.458336:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574427.458342:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574427.458347:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574427.458354:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574427.458361:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14893 00000400:00000200:0.0:1788574427.458371:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574427.458381:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574427.458388:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574427.458411:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458415:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458422:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458429:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574427.458434:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574427.458438:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574427.458443:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574427.458448:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574427.458454:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574427.458460:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574427.458468:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458473:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458480:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458484:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574427.458490:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574427.458496:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458501:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574427.458512:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574427.458517:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574427.458520:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458533:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574427.458540:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574427.458546:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458553:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574427.458561:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574427.458567:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574427.458572:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574427.458576:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458583:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574427.458589:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574427.458594:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574427.458603:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574427.458611:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.458623:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810265856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574427.458677:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574427.458681:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.458692:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810265856/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574427.458707:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574427.458712:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574427.458719:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574427.458723:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574427.458727:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574427.458734:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574427.458738:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14893 00000400:00000200:0.0:1788574427.458745:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574427.458750:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574427.458753:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574427.458769:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458774:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458782:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458786:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574427.458792:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574427.458795:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574427.458800:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574427.458805:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574427.458812:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574427.458816:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574427.458826:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458833:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.458840:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458845:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574427.458851:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574427.458857:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574427.458864:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574427.458871:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574427.458877:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574427.458882:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458888:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574427.458893:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574427.458901:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458908:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574427.458915:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574427.458923:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574427.458930:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574427.458935:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.458940:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574427.458944:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574427.458949:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574427.458958:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574427.458974:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.458987:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810265984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574427.459009:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574427.459015:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.459025:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810265984/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574427.459054:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574427.459061:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574427.459071:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574427.459076:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574427.459081:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574427.459088:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574427.459096:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14893 00000400:00000200:0.0:1788574427.459107:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574427.459117:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574427.459124:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574427.459133:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.459139:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.459145:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.459149:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574427.459155:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574427.459158:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574427.459162:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574427.459166:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574427.459172:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574427.459178:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574427.459188:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.459193:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574427.459207:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574427.459212:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574427.459220:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574427.459229:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574427.459236:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574427.459240:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574427.459244:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574427.459249:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.459254:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574427.459258:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574427.459264:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.459268:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574427.459275:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574427.459281:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574427.459288:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574427.459291:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574427.459296:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574427.459301:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574427.459306:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574427.459313:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574427.459321:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.459332:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810266112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574427.459351:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574427.459356:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.459365:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810266112/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574427.459413:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.459421:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14904) 00000020:00000001:0.0:1788574428.353782:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574428.353786:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574428.353790:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574429.377785:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574429.377790:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574429.377794:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574431.297743:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309570270, at 4309570304 00000100:00000010:0.0:1788574431.297758:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00080000:0.0:1788574432.577761:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574432.577772:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574432.577775:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574432.577779:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574432.577784:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574432.577793:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14899 00000400:00000200:0.0:1788574432.577801:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574432.577806:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574432.577810:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574432.577812:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.577815:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.577817:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.577818:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574432.577822:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574432.577824:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574432.577828:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574432.577832:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574432.577835:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574432.577838:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574432.577842:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.577844:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.577846:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.577848:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574432.577851:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574432.577855:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574432.577877:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574432.577881:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574432.577885:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574432.577888:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.577891:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574432.577893:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574432.577900:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574432.577904:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574432.577908:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574432.577910:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.577914:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574432.577917:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574432.577921:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574432.577930:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574432.577938:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.577959:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810266240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574432.577969:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574432.577972:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.577976:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810266240/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574432.577995:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574432.577997:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574432.578000:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574432.578001:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574432.578002:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574432.578004:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574432.578006:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14899 00000400:00000200:0.0:1788574432.578009:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574432.578011:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574432.578013:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574432.578015:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578022:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578024:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578026:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574432.578028:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574432.578030:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574432.578031:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574432.578033:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574432.578035:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574432.578036:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574432.578039:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578041:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578045:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578047:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574432.578058:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574432.578065:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578070:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574432.578074:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574432.578080:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574432.578084:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578087:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574432.578092:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574432.578096:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578102:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574432.578108:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574432.578112:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574432.578117:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574432.578119:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578122:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574432.578124:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574432.578128:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574432.578136:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574432.578140:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.578150:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810266368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574432.578160:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574432.578164:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.578169:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810266368/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574432.578175:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574432.578180:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574432.578184:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574432.578186:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574432.578189:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574432.578194:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574432.578201:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14899 00000400:00000200:0.0:1788574432.578213:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574432.578224:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574432.578229:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574432.578237:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578242:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578247:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578252:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574432.578257:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574432.578261:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574432.578265:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574432.578268:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574432.578271:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574432.578277:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574432.578291:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578296:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578300:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578311:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574432.578317:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574432.578323:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578326:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574432.578331:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574432.578337:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574432.578344:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578349:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574432.578355:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574432.578361:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578373:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574432.578382:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574432.578386:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574432.578390:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574432.578393:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578394:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574432.578395:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574432.578397:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574432.578403:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574432.578407:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.578415:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810266496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574432.578425:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574432.578429:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.578436:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810266496/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574432.578462:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574432.578466:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574432.578472:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574432.578475:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574432.578477:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574432.578486:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574432.578492:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14899 00000400:00000200:0.0:1788574432.578499:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574432.578503:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574432.578505:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574432.578511:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578513:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578515:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578525:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574432.578528:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574432.578533:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574432.578537:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574432.578540:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574432.578544:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574432.578547:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574432.578555:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578559:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578562:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578564:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574432.578567:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574432.578571:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578575:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574432.578578:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574432.578583:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574432.578587:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578590:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574432.578593:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574432.578597:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578601:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574432.578607:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574432.578620:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574432.578625:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574432.578630:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578667:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574432.578669:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574432.578671:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574432.578677:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574432.578683:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.578693:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810266624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574432.578707:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574432.578710:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.578716:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810266624/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574432.578728:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574432.578730:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574432.578733:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574432.578738:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574432.578743:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574432.578752:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574432.578758:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14899 00000400:00000200:0.0:1788574432.578766:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574432.578772:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574432.578778:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574432.578788:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578794:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578797:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578800:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574432.578803:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574432.578805:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574432.578817:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574432.578821:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574432.578826:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574432.578829:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574432.578834:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578837:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574432.578840:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578844:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574432.578849:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574432.578854:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574432.578855:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574432.578857:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574432.578860:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574432.578865:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578869:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574432.578872:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574432.578877:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578880:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574432.578886:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574432.578890:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574432.578893:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574432.578896:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574432.578899:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574432.578902:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574432.578906:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574432.578912:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574432.578919:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.578931:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810266752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574432.578950:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574432.578955:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.578961:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810266752/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574432.578982:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.578987:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14909) 00000020:00000001:0.0:1788574433.473786:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574433.473790:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574433.473795:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574434.497883:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574434.497887:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574434.497892:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574436.417715:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309575390, at 4309575424 00000100:00000010:0.0:1788574436.417729:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574438.593838:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574438.593846:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574438.593852:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574439.617838:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574439.617844:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574439.617847:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574441.537752:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309580510, at 4309580544 00000100:00000010:0.0:1788574441.537765:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788574443.713855:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574443.713858:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574443.713861:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574444.737832:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574444.737835:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574444.737838:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574447.681774:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309586655, at 4309586688 00000100:00000010:0.0:1788574447.681794:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788574448.833839:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574448.833844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574448.833847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574449.857853:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574449.857857:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574449.857860:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574452.801751:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309591774, at 4309591808 00000100:00000010:0.0:1788574452.801768:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788574453.953914:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574453.953920:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574453.953923:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574454.977801:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574454.977804:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574454.977807:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574457.921775:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309596895, at 4309596928 00000100:00000010:0.0:1788574457.921790:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788574459.073843:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574459.073846:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574459.073849:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574460.097844:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574460.097847:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574460.097850:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574463.041747:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309602014, at 4309602048 00000100:00000010:0.0:1788574463.041764:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788574464.193880:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574464.193883:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574464.193886:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574464.321762:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.321771:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574464.321774:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.321777:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.321783:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.321790:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14930 00000400:00000200:0.0:1788574464.321823:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574464.321832:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.321837:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.321842:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.321846:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.321863:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.321866:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574464.321869:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.321872:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574464.321874:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.321880:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574464.321884:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574464.321888:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574464.321893:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.321896:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.321901:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.321904:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574464.321908:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574464.321915:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574464.321920:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.321924:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.321930:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.321934:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.321938:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574464.321942:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.321949:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574464.321954:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574464.321959:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.321960:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.321963:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.321965:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.321969:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.321975:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574464.321981:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.321987:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810270848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.321995:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.322003:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.322007:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810270848/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.322026:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.322028:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.322031:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574464.322032:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.322033:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.322035:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.322037:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14930 00000400:00000200:0.0:1788574464.322040:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574464.322042:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.322044:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.322047:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322048:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322051:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322052:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574464.322054:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.322055:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574464.322056:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.322058:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574464.322060:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574464.322064:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574464.322067:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322071:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322077:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322079:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574464.322084:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574464.322090:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322104:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.322108:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.322115:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.322122:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322128:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.322135:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574464.322141:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322147:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.322153:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574464.322156:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574464.322161:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.322163:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322166:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.322168:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.322171:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.322176:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574464.322181:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.322188:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810270976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.322213:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.322220:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.322226:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810270976/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.322242:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.322244:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.322258:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574464.322260:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.322263:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.322266:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.322270:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14930 00000400:00000200:0.0:1788574464.322294:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574464.322307:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.322311:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.322314:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322319:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322325:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322330:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574464.322333:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.322335:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574464.322339:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.322341:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574464.322344:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574464.322347:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574464.322352:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322355:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322359:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322366:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574464.322368:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574464.322374:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322377:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.322380:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.322384:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.322389:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322392:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.322396:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574464.322399:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322402:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.322416:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574464.322421:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574464.322427:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.322429:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322434:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.322442:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.322445:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.322451:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574464.322454:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.322462:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810271104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.322475:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.322479:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.322484:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810271104/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.322507:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.322509:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.322517:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574464.322521:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.322524:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.322528:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.322534:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14930 00000400:00000200:0.0:1788574464.322543:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574464.322552:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.322559:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.322566:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322573:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322576:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322580:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574464.322582:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.322585:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574464.322587:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.322589:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574464.322592:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574464.322596:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574464.322607:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322612:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322615:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322620:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574464.322624:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574464.322629:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322634:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.322671:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.322674:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.322678:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322681:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.322684:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574464.322688:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322692:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.322697:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574464.322701:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574464.322704:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.322706:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322709:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.322712:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.322716:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.322723:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574464.322728:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.322735:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810271232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.322742:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.322746:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.322753:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810271232/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.322759:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.322761:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.322772:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574464.322776:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.322780:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.322785:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.322792:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14930 00000400:00000200:0.0:1788574464.322817:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574464.322825:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.322833:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.322840:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322847:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322853:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322859:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574464.322863:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.322865:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574464.322871:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.322877:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574464.322882:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574464.322889:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574464.322899:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322904:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.322910:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322914:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574464.322920:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574464.322928:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574464.322934:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.322940:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.322946:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.322950:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322955:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.322958:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574464.322971:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.322975:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.322981:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574464.322986:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574464.322993:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.322996:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.323002:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.323005:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.323009:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.323018:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574464.323026:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.323037:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810271360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.323062:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.323067:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.323077:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810271360/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.323093:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.323101:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14941) 00000020:00000001:0.0:1788574465.217856:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574465.217860:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574465.217862:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574465.666290:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574465.666295:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574465.666297:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574465.666300:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574465.666303:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574465.666306:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14936 00000400:00000200:0.0:1788574465.666313:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574465.666318:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574465.666321:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574465.666323:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666333:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666336:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666339:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574465.666343:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574465.666345:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574465.666346:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574465.666349:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574465.666352:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574465.666354:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574465.666357:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666359:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666362:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666364:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574465.666367:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574465.666370:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666373:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574465.666376:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574465.666379:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574465.666382:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666385:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574465.666387:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574465.666392:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574465.666395:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574465.666398:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574465.666400:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666402:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574465.666404:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574465.666407:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574465.666412:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574465.666415:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.666422:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810271616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574465.666443:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574465.666446:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.666452:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810271616/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574465.666475:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574465.666483:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574465.666487:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574465.666489:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574465.666491:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574465.666494:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574465.666497:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14936 00000400:00000200:0.0:1788574465.666502:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574465.666506:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574465.666509:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574465.666513:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666515:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666518:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666523:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574465.666529:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574465.666533:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574465.666538:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574465.666542:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574465.666548:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574465.666555:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574465.666562:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666568:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666574:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666575:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574465.666585:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574465.666589:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666596:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574465.666598:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574465.666605:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574465.666611:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666615:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574465.666620:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574465.666625:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666628:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574465.666632:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574465.666684:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574465.666691:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574465.666696:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666702:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574465.666705:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574465.666711:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574465.666719:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574465.666727:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.666737:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810271744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574465.666744:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574465.666746:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.666751:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810271744/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574465.666762:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574465.666765:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574465.666769:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574465.666770:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574465.666772:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574465.666776:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574465.666788:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14936 00000400:00000200:0.0:1788574465.666813:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574465.666831:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574465.666834:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574465.666840:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666841:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666847:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666852:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574465.666855:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574465.666858:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574465.666861:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574465.666863:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574465.666865:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574465.666866:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574465.666871:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666875:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.666879:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666882:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574465.666884:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574465.666890:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574465.666893:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574465.666895:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574465.666900:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574465.666904:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666907:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574465.666909:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574465.666914:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666919:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574465.666923:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574465.666926:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574465.666930:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574465.666938:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.666942:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574465.666945:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574465.666955:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574465.666961:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574465.666967:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.666976:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810271872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574465.666984:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574465.666987:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.666994:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810271872/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574465.667008:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574465.667011:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574465.667017:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574465.667020:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574465.667023:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574465.667026:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574465.667028:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14936 00000400:00000200:0.0:1788574465.667037:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574465.667044:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574465.667047:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574465.667051:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667056:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.667062:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667067:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574465.667071:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574465.667074:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574465.667076:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574465.667078:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574465.667081:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574465.667092:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574465.667098:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667102:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.667106:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667111:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574465.667116:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574465.667121:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667127:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574465.667130:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574465.667135:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574465.667139:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667142:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574465.667145:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574465.667151:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667155:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574465.667160:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574465.667164:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574465.667170:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574465.667173:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667176:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574465.667179:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574465.667182:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574465.667187:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574465.667191:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.667200:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810272000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574465.667208:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574465.667212:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.667217:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810272000/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574465.667230:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574465.667238:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574465.667244:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574465.667248:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574465.667250:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574465.667257:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574465.667261:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14936 00000400:00000200:0.0:1788574465.667268:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574465.667272:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574465.667276:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574465.667282:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667287:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.667290:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667294:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574465.667299:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574465.667301:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574465.667305:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574465.667308:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574465.667313:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574465.667315:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574465.667320:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667324:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574465.667329:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667336:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574465.667343:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574465.667351:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574465.667356:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574465.667361:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574465.667368:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574465.667374:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667380:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574465.667390:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574465.667397:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667405:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574465.667413:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574465.667419:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574465.667422:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574465.667423:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.667425:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574465.667426:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574465.667428:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574465.667434:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574465.667439:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.667448:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810272128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574465.667461:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574465.667465:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.667473:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810272128/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574465.667485:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574465.667492:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14942) 00000020:00000001:0.0:1788574469.313802:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574469.313805:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574469.313808:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574469.505741:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309608479, at 4309608512 00000100:00000010:0.0:1788574469.505763:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788574470.337842:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574470.337847:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574470.337851:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574474.433831:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574474.433835:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574474.433838:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574474.817736:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309613790, at 4309613824 00000100:00000010:0.0:1788574474.817751:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788574475.457875:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574475.457878:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574475.457892:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574476.097796:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574476.097807:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574476.097810:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574476.097814:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574476.097819:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574476.097827:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 14942 00000400:00000200:0.0:1788574476.097836:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574476.097842:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574476.097846:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574476.097849:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.097853:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.097856:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.097858:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574476.097862:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574476.097865:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574476.097868:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574476.097871:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574476.097874:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574476.097877:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574476.097880:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.097882:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.097885:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.097887:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574476.097890:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574476.097894:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574476.097898:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574476.097901:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574476.097921:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574476.097924:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.097927:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574476.097930:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574476.097955:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574476.097960:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574476.097964:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574476.097966:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.097969:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574476.097973:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574476.097977:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574476.097996:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574476.098004:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.098015:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810272896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574476.098026:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574476.098030:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.098035:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810272896/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574476.098059:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574476.098062:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574476.098066:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574476.098068:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574476.098070:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574476.098073:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574476.098077:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14942 00000400:00000200:0.0:1788574476.098082:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574476.098088:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574476.098091:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574476.098095:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098100:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098113:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098119:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574476.098126:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574476.098130:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574476.098135:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574476.098138:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574476.098143:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574476.098149:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574476.098157:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098162:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098170:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098175:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574476.098182:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574476.098190:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098197:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574476.098203:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574476.098211:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574476.098219:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098222:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574476.098230:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574476.098237:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098242:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574476.098252:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574476.098259:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574476.098266:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574476.098270:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098274:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574476.098276:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574476.098281:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574476.098289:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574476.098297:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.098308:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810273024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574476.098333:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574476.098337:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.098345:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810273024/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574476.098361:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574476.098366:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574476.098371:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574476.098376:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574476.098378:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574476.098383:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574476.098389:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14942 00000400:00000200:0.0:1788574476.098394:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574476.098400:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574476.098406:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574476.098415:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098420:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098424:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098429:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574476.098433:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574476.098435:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574476.098441:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574476.098444:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574476.098449:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574476.098454:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574476.098462:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098467:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098472:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098475:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574476.098487:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574476.098495:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098501:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574476.098504:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574476.098509:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574476.098514:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098520:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574476.098524:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574476.098529:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098533:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574476.098539:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574476.098544:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574476.098550:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574476.098554:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098559:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574476.098563:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574476.098569:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574476.098575:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574476.098581:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.098592:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810273152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574476.098609:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574476.098615:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.098624:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810273152/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574476.098712:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574476.098720:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574476.098730:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574476.098733:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574476.098738:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574476.098743:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574476.098756:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14942 00000400:00000200:0.0:1788574476.098766:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574476.098772:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574476.098774:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574476.098781:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098786:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098793:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098799:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574476.098804:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574476.098807:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574476.098811:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574476.098814:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574476.098819:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574476.098824:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574476.098831:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098834:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.098840:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098847:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574476.098852:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574476.098862:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574476.098867:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574476.098871:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574476.098879:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574476.098883:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098888:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574476.098895:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574476.098900:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098904:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574476.098910:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574476.098915:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574476.098922:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574476.098948:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.098953:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574476.098958:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574476.098963:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574476.098971:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574476.098979:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.098989:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810273280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574476.099005:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574476.099007:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.099012:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810273280/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574476.099030:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574476.099034:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574476.099040:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574476.099046:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574476.099051:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574476.099058:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574476.099062:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 14942 00000400:00000200:0.0:1788574476.099069:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574476.099074:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574476.099078:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574476.099085:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.099089:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.099094:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.099097:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574476.099103:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574476.099105:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574476.099109:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574476.099111:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574476.099117:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574476.099130:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574476.099137:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.099142:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574476.099149:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574476.099155:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574476.099160:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574476.099167:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574476.099173:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574476.099179:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574476.099186:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574476.099192:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.099198:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574476.099202:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574476.099207:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.099212:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574476.099217:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574476.099224:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574476.099230:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574476.099231:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.099233:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574476.099234:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574476.099236:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574476.099240:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574476.099245:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.099256:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810273408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574476.099272:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574476.099277:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.099286:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810273408/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574476.099306:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574476.099321:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14952) 00000020:00000001:0.0:1788574479.553836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574479.553840:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574479.553843:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574479.937761:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309618910, at 4309618944 00000100:00000010:0.0:1788574479.937785:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788574480.577861:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574480.577867:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574480.577871:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574484.675059:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574484.675063:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574484.675066:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574485.057724:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309624031, at 4309624064 00000100:00000010:0.0:1788574485.057744:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574485.697897:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574485.697901:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574485.697904:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574487.809951:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574489.793836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574489.793840:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574489.793843:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574490.753775:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309629727, at 4309629760 00000100:00000010:0.0:1788574490.753784:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000020:00000001:0.0:1788574490.817874:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574490.817879:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574490.817883:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574494.913788:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574494.913793:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574494.913797:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574495.809734:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309634782, at 4309634816 00000100:00000010:0.0:1788574495.809752:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574495.937825:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574495.937829:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574495.937832:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574500.033827:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574500.033830:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574500.033833:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574500.929751:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309639902, at 4309639936 00000100:00000010:0.0:1788574500.929765:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788574501.057839:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574501.057843:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574501.057846:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574505.153835:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574505.153839:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574505.153842:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574506.049742:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309645022, at 4309645056 00000100:00000010:0.0:1788574506.049758:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788574506.177829:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574506.177834:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574506.177837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574510.273830:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574510.273834:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574510.273836:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574511.297863:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574511.297866:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574511.297869:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574513.025775:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309651999, at 4309652032 00000100:00000010:0.0:1788574513.025796:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788574515.393885:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574515.393890:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574515.393894:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574516.417800:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574516.417805:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574516.417808:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574518.337743:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309657310, at 4309657344 00000100:00000010:0.0:1788574518.337760:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788574520.513780:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574520.513785:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574520.513817:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574521.537831:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574521.537834:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574521.537837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574523.457727:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309662430, at 4309662464 00000100:00000010:0.0:1788574523.457748:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788574525.633879:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574525.633885:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574525.633890:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574526.657827:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574526.657830:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574526.657833:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574526.721898:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574526.721904:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574526.721913:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574526.721917:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574526.721919:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574528.577750:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309667551, at 4309667584 00000100:00000010:0.0:1788574528.577766:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788574530.562239:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.562244:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574530.562246:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.562250:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.562253:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.562256:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15001 00000400:00000200:0.0:1788574530.562263:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574530.562268:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.562270:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.562273:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562275:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562277:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562279:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574530.562281:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.562289:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574530.562291:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.562293:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574530.562294:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574530.562296:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574530.562298:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562300:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562301:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562302:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574530.562304:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574530.562307:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562309:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.562311:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.562313:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.562315:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562316:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574530.562318:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.562349:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574530.562352:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574530.562354:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.562355:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562357:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.562358:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.562360:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.562365:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574530.562369:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.562377:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810281088/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.562386:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.562389:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.562394:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810281088/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.562432:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.562436:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.562439:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574530.562440:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.562442:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.562444:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.562446:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15001 00000400:00000200:0.0:1788574530.562449:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574530.562452:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.562454:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.562456:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562458:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562459:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562461:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574530.562463:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.562464:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574530.562465:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.562467:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574530.562469:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574530.562471:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574530.562473:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562475:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562480:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562483:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574530.562488:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574530.562493:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562495:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.562504:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.562517:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.562522:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562526:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.562532:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574530.562536:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562539:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.562544:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574530.562548:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574530.562550:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.562553:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562556:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.562557:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.562561:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.562568:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574530.562573:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.562581:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810281216/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.562595:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.562599:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.562610:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810281216/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.562629:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.562668:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.562674:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574530.562678:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.562681:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.562685:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.562690:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15001 00000400:00000200:0.0:1788574530.562722:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574530.562729:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.562732:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.562744:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562748:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562753:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562756:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574530.562763:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.562768:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574530.562773:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.562779:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574530.562786:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574530.562791:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574530.562799:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562804:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.562811:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562818:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574530.562825:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574530.562830:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562834:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.562839:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.562843:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.562847:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562850:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.562852:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574530.562853:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562854:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.562860:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574530.562862:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574530.562865:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.562868:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.562871:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.562872:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.562874:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.562878:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574530.562889:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.562895:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810281344/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.562902:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.562906:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.562910:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810281344/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.562919:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.562924:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.562929:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574530.562932:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.562936:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.562947:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.562954:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15001 00000400:00000200:0.0:1788574530.562962:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574530.562968:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.562973:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.562980:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.562986:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.563003:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563008:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574530.563014:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.563016:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574530.563019:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.563023:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574530.563027:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574530.563030:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574530.563035:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563039:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.563049:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563052:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574530.563059:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574530.563065:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563068:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.563069:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.563072:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.563077:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563082:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.563085:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574530.563088:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563093:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.563100:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574530.563103:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574530.563107:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.563109:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563113:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.563115:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.563119:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.563124:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574530.563129:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.563133:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810281472/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.563144:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.563148:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.563156:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810281472/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.563179:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.563184:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.563189:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574530.563192:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.563202:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.563209:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.563214:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15001 00000400:00000200:0.0:1788574530.563221:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574530.563227:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.563230:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.563234:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563239:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.563242:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563244:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574530.563245:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.563246:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574530.563247:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.563249:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574530.563253:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574530.563255:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574530.563257:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563259:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.563261:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563265:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574530.563270:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574530.563273:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574530.563278:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.563282:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.563284:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.563287:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563288:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.563291:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574530.563293:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563297:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.563305:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574530.563310:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574530.563313:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.563316:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.563320:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.563336:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.563340:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.563347:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574530.563355:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.563363:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810281600/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.563370:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.563374:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.563380:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810281600/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.563396:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.563401:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15007) 00000020:00000001:0.0:1788574530.753834:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574530.753839:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574530.753843:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574531.777834:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574531.777837:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574531.777839:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574534.401746:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309673375, at 4309673408 00000100:00000010:0.0:1788574534.401762:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788574535.873793:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574535.873798:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574535.873802:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574536.897849:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574536.897853:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574536.897856:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574539.841801:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309678814, at 4309678848 00000100:00000010:0.0:1788574539.841817:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788574540.993807:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574540.993811:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574540.993813:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574542.017915:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574542.017923:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574542.017928:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788574544.193966:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788574544.194024:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574544.194034:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574544.194039:0:2391:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788574544.194045:0:2391:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574544.194050:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574544.194057:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574544.194061:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9400. 00000400:00000200:0.0:1788574544.194068:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574544.194072:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574544.194079:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3f68 00000400:00000010:0.0:1788574544.194081:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3f68. 00000100:00000001:0.0:1788574544.194086:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574544.194094:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6bb80 x1875440810282368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194120:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574544.194125:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810282368/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194133:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.194135:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574544.194145:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574544.194148:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9800. 00000400:00000200:0.0:1788574544.194152:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574544.194157:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574544.194161:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3bb0 00000400:00000010:0.0:1788574544.194163:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3bb0. 00000100:00000001:0.0:1788574544.194183:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574544.194189:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69f80 x1875440810282496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194202:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574544.194210:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810282496/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194228:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.194231:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574544.194241:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574544.194247:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788574544.194251:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574544.194259:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574544.194264:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3110 00000400:00000010:0.0:1788574544.194270:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3110. 00000100:00000001:0.0:1788574544.194274:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574544.194281:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69880 x1875440810282624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194304:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574544.194311:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810282624/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194327:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.194331:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574544.194341:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574544.194346:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788574544.194351:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574544.194363:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574544.194369:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3908 00000400:00000010:0.0:1788574544.194373:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3908. 00000100:00000001:0.0:1788574544.194377:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574544.194388:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68000 x1875440810282752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194440:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574544.194448:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810282752/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194463:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.194468:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574544.194480:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574544.194483:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788574544.194486:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574544.194492:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574544.194496:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f34c8 00000400:00000010:0.0:1788574544.194497:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f34c8. 00000100:00000001:0.0:1788574544.194504:0:2391:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574544.194513:0:2391:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69c00 x1875440810282880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194537:0:2391:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574544.194545:0:2391:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810282880/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574544.194559:0:2391:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.194570:0:2391:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788574544.961706:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309683934, at 4309683968 00000100:00000010:0.0:1788574544.961721:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574546.113796:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574546.113799:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574546.113802:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574547.137839:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574547.137844:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574547.137848:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574550.081714:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309689054, at 4309689088 00000100:00000010:0.0:1788574550.081729:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788574551.233785:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574551.233806:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574551.233810:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574552.257869:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574552.257873:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574552.257878:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574552.258513:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574552.258519:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574552.258520:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574552.258523:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574552.258526:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574552.258530:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574552.258537:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574552.258542:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574552.258544:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574552.258547:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574552.258549:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574552.258551:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574552.258552:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574552.258555:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574552.258556:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574552.258557:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574552.258559:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574552.258561:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574552.258563:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574552.258564:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574552.258566:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574552.258567:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574552.258568:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574552.258570:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574552.258572:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574552.258574:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574552.258583:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574552.258587:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574552.258589:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574552.258592:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574552.258594:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574552.258599:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574552.258601:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574552.258604:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574552.258606:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574552.258609:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574552.258611:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574552.258613:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574552.258619:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574552.258623:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574552.258630:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810284416/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574552.258662:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574552.258665:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574552.258670:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810284416/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574552.258688:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574552.258693:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15028) 00000100:00000040:0.0:1788574556.097765:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309695071, at 4309695104 00000100:00000010:0.0:1788574556.097783:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788574556.353847:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574556.353852:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574556.353856:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574557.377772:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574557.377779:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574557.377783:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574557.505742:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574557.505751:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574557.505753:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574557.505773:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574557.505781:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574557.505788:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574557.505796:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574557.505802:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574557.505804:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574557.505807:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505809:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.505811:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505813:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574557.505816:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574557.505818:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574557.505820:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574557.505822:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574557.505824:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574557.505825:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574557.505827:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505828:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.505830:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505831:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574557.505834:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574557.505836:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505838:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574557.505840:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574557.505843:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574557.505845:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.505846:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574557.505848:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574557.505853:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574557.505855:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574557.505858:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574557.505863:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.505866:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574557.505868:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574557.505872:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574557.505878:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574557.505884:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.505891:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810284544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574557.505899:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574557.505903:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.505906:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810284544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574557.505927:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574557.505930:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574557.505932:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574557.505933:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574557.505935:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574557.505936:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574557.505938:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574557.505941:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574557.505943:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574557.505944:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574557.505946:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505948:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.505949:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505950:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574557.505952:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574557.505952:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574557.505953:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574557.505955:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574557.505956:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574557.505961:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574557.505965:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505967:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.505968:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505970:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574557.505972:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574557.505974:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574557.505975:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574557.505977:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574557.505982:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574557.505985:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.505988:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574557.505993:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574557.505996:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.505999:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574557.506004:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574557.506008:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574557.506012:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574557.506013:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506014:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574557.506015:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574557.506017:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574557.506024:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574557.506028:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.506035:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810284672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574557.506047:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574557.506050:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.506057:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810284672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574557.506066:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574557.506075:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574557.506079:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574557.506080:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574557.506082:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574557.506085:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574557.506088:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574557.506096:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574557.506099:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574557.506101:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574557.506106:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506110:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506114:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506115:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574557.506117:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574557.506120:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574557.506122:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574557.506124:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574557.506127:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574557.506133:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574557.506138:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506140:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506144:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506149:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574557.506153:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574557.506158:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506161:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574557.506164:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574557.506168:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574557.506171:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506174:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574557.506183:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574557.506186:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506190:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574557.506195:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574557.506199:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574557.506203:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574557.506206:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506209:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574557.506211:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574557.506215:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574557.506220:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574557.506226:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.506236:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810284800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574557.506252:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574557.506255:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.506263:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810284800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574557.506273:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574557.506278:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574557.506283:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574557.506287:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574557.506288:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574557.506290:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574557.506292:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574557.506296:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574557.506302:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574557.506305:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574557.506310:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506315:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506320:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506331:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574557.506335:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574557.506337:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574557.506338:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574557.506340:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574557.506344:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574557.506347:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574557.506350:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506355:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506357:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506361:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574557.506372:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574557.506382:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506387:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574557.506390:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574557.506393:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574557.506396:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506399:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574557.506402:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574557.506407:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506412:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574557.506418:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574557.506423:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574557.506426:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574557.506427:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506429:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574557.506432:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574557.506435:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574557.506441:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574557.506446:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.506477:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810284928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574557.506502:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574557.506507:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.506515:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810284928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574557.506534:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574557.506537:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574557.506545:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574557.506548:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574557.506550:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574557.506554:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574557.506557:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15023 00000400:00000200:0.0:1788574557.506563:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574557.506569:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574557.506574:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574557.506579:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506583:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506587:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506590:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574557.506592:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574557.506595:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574557.506597:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574557.506600:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574557.506606:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574557.506611:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574557.506617:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506622:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574557.506626:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506630:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574557.506666:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574557.506672:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574557.506678:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574557.506681:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574557.506683:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574557.506685:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506686:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574557.506688:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574557.506694:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506696:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574557.506702:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574557.506706:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574557.506711:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574557.506714:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574557.506717:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574557.506719:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574557.506722:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574557.506728:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574557.506732:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.506738:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810285056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574557.506750:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574557.506752:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.506755:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810285056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574557.506762:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574557.506769:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15034) 00000100:00000040:0.0:1788574561.345730:0:267754:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309700318, at 4309700352 00000100:00000010:0.0:1788574561.345745:0:267754:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788574561.473841:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574561.473845:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574561.473847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574562.497761:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574562.497765:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574562.497768:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:0.0:1788574564.610033:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574564.610055:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574564.610061:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574564.610063:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574566.465709:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309705438, at 4309705472 00000100:00000010:0.0:1788574566.465727:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788574566.593837:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574566.593841:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574566.593844:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574567.617806:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574567.617811:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574567.617814:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574567.745816:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574567.745830:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574567.745834:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574567.745840:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574567.745845:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574567.745854:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15034 00000400:00000200:0.0:1788574567.745866:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574567.745874:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574567.745878:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574567.745882:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.745886:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.745889:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.745892:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574567.745897:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574567.745902:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574567.745906:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574567.745910:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574567.745913:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574567.745939:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574567.745943:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.745946:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.745948:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.745951:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574567.745954:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574567.745958:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574567.745961:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574567.745965:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574567.745969:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574567.745972:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.745975:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574567.745978:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574567.745987:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574567.745991:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574567.745997:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574567.745999:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746003:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574567.746007:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574567.746012:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574567.746022:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574567.746031:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.746042:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810285824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574567.746056:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574567.746061:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.746067:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810285824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574567.746094:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574567.746098:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574567.746102:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574567.746109:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574567.746111:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574567.746114:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574567.746117:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15034 00000400:00000200:0.0:1788574567.746123:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574567.746128:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574567.746131:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574567.746135:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746138:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746142:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746146:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574567.746149:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574567.746153:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574567.746158:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574567.746161:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574567.746164:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574567.746168:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574567.746176:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746181:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746185:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746189:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574567.746193:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574567.746199:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746206:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574567.746212:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574567.746221:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574567.746227:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746231:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574567.746238:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574567.746244:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746254:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574567.746261:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574567.746265:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574567.746271:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574567.746275:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746277:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574567.746279:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574567.746284:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574567.746291:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574567.746298:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.746305:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810285952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574567.746319:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574567.746324:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.746329:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810285952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574567.746344:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574567.746350:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574567.746358:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574567.746363:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574567.746366:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574567.746373:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574567.746378:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15034 00000400:00000200:0.0:1788574567.746386:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574567.746408:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574567.746414:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574567.746421:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746424:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746427:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746433:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574567.746438:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574567.746448:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574567.746450:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574567.746455:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574567.746460:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574567.746464:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574567.746475:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746481:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746487:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746494:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574567.746501:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574567.746509:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746516:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574567.746520:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574567.746524:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574567.746530:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746535:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574567.746538:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574567.746543:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746546:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574567.746551:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574567.746556:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574567.746559:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574567.746562:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746567:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574567.746571:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574567.746575:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574567.746581:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574567.746588:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.746599:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810286080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574567.746611:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574567.746616:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.746629:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810286080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574567.746687:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574567.746693:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574567.746700:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574567.746703:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574567.746707:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574567.746713:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574567.746719:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15034 00000400:00000200:0.0:1788574567.746725:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574567.746733:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574567.746738:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574567.746745:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746750:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746755:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746761:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574567.746766:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574567.746769:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574567.746774:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574567.746779:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574567.746784:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574567.746787:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574567.746791:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746798:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.746802:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746807:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574567.746811:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574567.746818:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574567.746824:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574567.746836:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574567.746841:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574567.746845:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746849:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574567.746853:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574567.746858:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746862:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574567.746871:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574567.746875:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574567.746880:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574567.746883:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.746888:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574567.746890:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574567.746893:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574567.746900:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574567.746906:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.746915:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810286208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574567.746933:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574567.746937:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.746948:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810286208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574567.746966:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574567.746973:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574567.746979:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574567.746984:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574567.746987:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574567.746993:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574567.746998:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15034 00000400:00000200:0.0:1788574567.747003:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574567.747009:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574567.747014:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574567.747029:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.747033:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.747038:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.747042:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574567.747049:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574567.747052:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574567.747056:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574567.747060:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574567.747066:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574567.747072:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574567.747081:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.747086:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574567.747091:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574567.747096:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574567.747101:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574567.747107:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574567.747110:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574567.747112:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574567.747116:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574567.747122:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.747127:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574567.747132:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574567.747139:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.747144:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574567.747151:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574567.747156:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574567.747161:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574567.747165:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.747172:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574567.747176:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574567.747181:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574567.747189:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574567.747203:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.747211:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810286336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574567.747227:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574567.747232:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.747240:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810286336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574567.747256:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574567.747264:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15044) 00000100:00000040:0.0:1788574571.585750:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309710559, at 4309710592 00000100:00000010:0.0:1788574571.585768:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574571.713890:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574571.713896:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574571.713900:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574572.737857:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574572.737865:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574572.737869:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574572.865702:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574572.865713:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574572.865716:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574572.865720:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574572.865724:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574572.865732:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15039 00000400:00000200:0.0:1788574572.865741:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574572.865747:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574572.865751:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574572.865755:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865758:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.865761:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865764:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574572.865768:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574572.865785:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574572.865788:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574572.865791:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574572.865794:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574572.865797:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574572.865801:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865803:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.865805:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865807:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574572.865810:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574572.865814:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865817:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574572.865820:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574572.865824:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574572.865827:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.865829:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574572.865832:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574572.865838:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574572.865842:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574572.865846:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574572.865848:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.865851:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574572.865853:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574572.865857:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574572.865865:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574572.865873:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.865881:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810286464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574572.865893:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574572.865897:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.865902:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810286464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574572.865939:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574572.865943:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574572.865946:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574572.865948:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574572.865949:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574572.865952:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574572.865955:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15039 00000400:00000200:0.0:1788574572.865959:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574572.865962:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574572.865965:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574572.865968:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865971:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.865974:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865977:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574572.865979:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574572.865981:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574572.865983:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574572.865985:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574572.865988:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574572.865990:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574572.865993:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.865996:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.865999:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866002:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574572.866006:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574572.866011:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866016:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574572.866024:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574572.866032:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574572.866037:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866041:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574572.866047:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574572.866053:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866058:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574572.866064:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574572.866068:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574572.866070:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574572.866074:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866077:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574572.866080:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574572.866084:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574572.866092:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574572.866099:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.866109:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810286592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574572.866123:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574572.866127:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.866135:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810286592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574572.866151:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574572.866157:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574572.866161:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574572.866164:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574572.866168:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574572.866170:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574572.866174:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15039 00000400:00000200:0.0:1788574572.866182:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574572.866189:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574572.866194:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574572.866207:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866212:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866218:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866223:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574572.866227:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574572.866232:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574572.866234:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574572.866237:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574572.866241:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574572.866246:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574572.866253:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866256:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866260:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866264:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574572.866268:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574572.866274:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866279:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574572.866283:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574572.866287:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574572.866291:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866295:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574572.866298:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574572.866301:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866303:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574572.866307:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574572.866313:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574572.866318:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574572.866320:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866324:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574572.866327:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574572.866330:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574572.866336:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574572.866347:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.866356:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810286720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574572.866542:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574572.866547:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.866555:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810286720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574572.866584:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574572.866588:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574572.866595:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574572.866598:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574572.866602:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574572.866607:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574572.866612:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15039 00000400:00000200:0.0:1788574572.866619:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574572.866625:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574572.866629:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574572.866656:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866659:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866662:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866665:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574572.866668:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574572.866669:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574572.866673:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574572.866676:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574572.866680:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574572.866685:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574572.866694:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866700:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866714:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866718:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574572.866723:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574572.866728:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866733:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574572.866737:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574572.866742:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574572.866748:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866753:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574572.866757:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574572.866762:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866767:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574572.866772:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574572.866778:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574572.866783:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574572.866786:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.866789:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574572.866793:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574572.866797:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574572.866803:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574572.866811:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.866823:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810286848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574572.866838:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574572.866841:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.866850:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810286848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574572.866865:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574572.866870:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574572.866876:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574572.866880:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574572.866890:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574572.866897:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574572.866903:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15039 00000400:00000200:0.0:1788574572.866911:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574572.866919:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574572.866923:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574572.866930:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866934:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866940:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866945:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574572.866950:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574572.866953:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574572.866957:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574572.866960:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574572.866963:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574572.866966:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574572.866973:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866978:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574572.866985:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574572.866990:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574572.866996:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574572.867004:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574572.867009:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574572.867013:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574572.867018:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574572.867025:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.867030:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574572.867034:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574572.867039:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.867041:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574572.867053:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574572.867059:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574572.867064:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574572.867068:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.867072:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574572.867076:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574572.867080:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574572.867087:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574572.867094:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.867105:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810286976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574572.867119:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574572.867124:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.867134:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810286976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574572.867155:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574572.867163:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15049) 00000100:00080000:0.0:1788574573.698665:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574573.698677:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574573.698680:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574573.698685:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574573.698691:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574573.698700:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15044 00000400:00000200:0.0:1788574573.698709:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574573.698716:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574573.698719:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574573.698723:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698726:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.698729:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698731:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574573.698735:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574573.698753:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574573.698756:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574573.698760:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574573.698763:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574573.698765:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574573.698769:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698771:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.698774:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698776:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574573.698779:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574573.698783:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698786:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574573.698789:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574573.698792:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574573.698795:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.698798:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574573.698800:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574573.698806:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574573.698810:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574573.698815:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574573.698816:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.698819:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574573.698823:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574573.698827:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574573.698834:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574573.698841:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.698850:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810287232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574573.698870:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574573.698875:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.698880:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810287232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574573.698917:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574573.698920:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574573.698924:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574573.698926:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574573.698934:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574573.698937:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574573.698941:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15044 00000400:00000200:0.0:1788574573.698947:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574573.698951:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574573.698955:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574573.698960:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698964:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.698969:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.698974:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574573.698978:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574573.698980:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574573.698985:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574573.698988:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574573.698994:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574573.699000:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574573.699007:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699013:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699016:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699020:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574573.699027:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574573.699036:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699041:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574573.699050:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574573.699059:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574573.699063:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699066:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574573.699073:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574573.699078:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699081:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574573.699088:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574573.699093:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574573.699096:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574573.699098:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699101:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574573.699103:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574573.699105:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574573.699110:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574573.699115:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.699122:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810287360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574573.699138:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574573.699142:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.699152:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810287360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574573.699169:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574573.699174:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574573.699179:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574573.699183:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574573.699185:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574573.699190:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574573.699205:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15044 00000400:00000200:0.0:1788574573.699264:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574573.699271:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574573.699274:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574573.699285:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699296:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699302:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699306:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574573.699311:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574573.699314:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574573.699319:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574573.699323:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574573.699327:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574573.699331:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574573.699339:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699343:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699347:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699351:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574573.699366:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574573.699379:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699386:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574573.699391:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574573.699396:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574573.699401:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699403:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574573.699405:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574573.699408:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699410:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574573.699414:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574573.699418:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574573.699424:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574573.699427:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699431:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574573.699434:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574573.699437:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574573.699450:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574573.699463:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.699473:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810287488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574573.699488:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574573.699492:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.699501:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810287488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574573.699513:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574573.699516:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574573.699520:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574573.699521:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574573.699523:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574573.699526:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574573.699530:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15044 00000400:00000200:0.0:1788574573.699537:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574573.699544:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574573.699548:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574573.699552:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699557:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699563:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699568:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574573.699572:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574573.699575:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574573.699577:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574573.699579:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574573.699584:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574573.699586:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574573.699590:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699596:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699607:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699612:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574573.699616:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574573.699623:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699628:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574573.699632:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574573.699659:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574573.699663:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699665:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574573.699669:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574573.699673:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699675:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574573.699681:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574573.699686:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574573.699690:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574573.699694:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699699:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574573.699702:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574573.699706:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574573.699713:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574573.699720:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.699731:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810287616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574573.699744:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574573.699748:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.699757:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810287616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574573.699771:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574573.699777:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574573.699785:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574573.699788:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574573.699796:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574573.699802:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574573.699807:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15044 00000400:00000200:0.0:1788574573.699813:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574573.699818:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574573.699821:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574573.699825:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699830:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699835:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699840:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574573.699845:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574573.699847:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574573.699849:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574573.699851:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574573.699853:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574573.699855:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574573.699860:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699864:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574573.699870:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699875:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574573.699880:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574573.699889:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574573.699893:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574573.699898:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574573.699903:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574573.699906:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699909:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574573.699912:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574573.699919:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699924:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574573.699934:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574573.699940:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574573.699946:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574573.699949:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574573.699953:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574573.699956:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574573.699960:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574573.699967:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574573.699975:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.699986:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810287744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574573.700001:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574573.700007:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.700013:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810287744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574573.700031:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574573.700039:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15050) 00000020:00000001:0.0:1788574576.833802:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574576.833806:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574576.833808:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574577.537746:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309716511, at 4309716544 00000100:00000010:0.0:1788574577.537761:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788574577.857876:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574577.857880:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574577.857884:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574579.009825:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574579.009841:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574579.009845:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574579.009851:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574579.009857:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574579.009867:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15045 00000400:00000200:0.0:1788574579.009880:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574579.009916:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574579.009921:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574579.009926:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.009930:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.009932:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.009935:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574579.009940:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574579.009943:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574579.009947:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574579.009951:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574579.009954:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574579.009956:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574579.009960:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.009963:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.009966:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.009968:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574579.009972:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574579.009976:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574579.009979:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574579.009983:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574579.009988:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574579.009992:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.009995:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574579.009998:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574579.010006:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574579.010011:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574579.010016:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574579.010018:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010021:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574579.010025:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574579.010030:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574579.010040:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574579.010056:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.010069:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810287872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574579.010082:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574579.010088:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.010094:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810287872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574579.010127:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574579.010131:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574579.010135:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574579.010137:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574579.010139:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574579.010142:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574579.010145:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15045 00000400:00000200:0.0:1788574579.010150:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574579.010155:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574579.010158:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574579.010162:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010165:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010168:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010171:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574579.010176:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574579.010180:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574579.010185:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574579.010188:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574579.010192:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574579.010196:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574579.010205:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010210:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010221:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010226:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574579.010231:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574579.010236:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010241:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574579.010245:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574579.010253:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574579.010257:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010261:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574579.010267:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574579.010273:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010277:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574579.010283:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574579.010288:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574579.010293:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574579.010296:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010300:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574579.010304:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574579.010306:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574579.010314:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574579.010320:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.010344:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810288000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574579.010355:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574579.010360:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.010367:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810288000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574579.010390:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574579.010393:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574579.010399:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574579.010401:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574579.010414:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574579.010421:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574579.010427:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15045 00000400:00000200:0.0:1788574579.010473:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574579.010481:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574579.010497:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574579.010504:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010509:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010514:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010519:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574579.010526:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574579.010529:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574579.010533:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574579.010536:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574579.010538:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574579.010539:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574579.010542:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010544:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010548:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010551:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574579.010553:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574579.010557:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010564:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574579.010568:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574579.010573:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574579.010580:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010583:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574579.010587:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574579.010593:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010598:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574579.010609:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574579.010614:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574579.010619:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574579.010622:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010627:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574579.010630:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574579.010664:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574579.010670:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574579.010676:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.010685:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810288128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574579.010695:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574579.010697:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.010701:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810288128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574579.010710:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574579.010712:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574579.010717:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574579.010721:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574579.010723:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574579.010727:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574579.010731:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15045 00000400:00000200:0.0:1788574579.010737:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574579.010741:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574579.010746:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574579.010753:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010758:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010762:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010766:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574579.010768:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574579.010776:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574579.010779:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574579.010783:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574579.010788:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574579.010792:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574579.010796:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010800:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010803:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010807:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574579.010812:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574579.010817:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010819:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574579.010820:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574579.010822:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574579.010825:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010828:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574579.010830:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574579.010836:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010840:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574579.010844:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574579.010848:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574579.010852:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574579.010855:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.010860:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574579.010862:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574579.010865:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574579.010868:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574579.010871:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.010875:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810288256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574579.010894:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574579.010897:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.010909:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810288256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574579.010919:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574579.010924:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574579.010930:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574579.010932:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574579.010934:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574579.010938:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574579.010941:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15045 00000400:00000200:0.0:1788574579.010944:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574579.010946:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574579.010949:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574579.010954:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010956:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010960:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010964:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574579.010968:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574579.010971:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574579.010973:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574579.010977:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574579.010980:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574579.010983:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574579.010989:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010991:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574579.010993:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574579.010994:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574579.010995:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574579.010997:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574579.011000:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574579.011006:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574579.011011:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574579.011016:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.011020:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574579.011024:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574579.011029:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.011034:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574579.011036:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574579.011038:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574579.011044:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574579.011045:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.011049:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574579.011050:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574579.011052:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574579.011058:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574579.011063:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.011070:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810288384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574579.011079:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574579.011082:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.011085:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810288384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574579.011095:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574579.011100:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15055) 00000020:00000001:0.0:1788574581.953881:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574581.953886:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574581.953890:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574582.849760:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309721823, at 4309721856 00000100:00000010:0.0:1788574582.849777:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788574582.977842:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574582.977846:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574582.977849:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574587.073856:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574587.073866:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574587.073868:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574587.969712:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309726942, at 4309726976 00000100:00000010:0.0:1788574587.969727:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788574588.097835:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574588.097840:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574588.097843:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574592.193892:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574592.193896:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574592.193899:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574593.089705:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309732062, at 4309732096 00000100:00000010:0.0:1788574593.089719:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788574593.217825:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574593.217828:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574593.217831:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574594.369784:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574594.369794:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574594.369796:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574594.369800:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574594.369803:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574594.369810:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15060 00000400:00000200:0.0:1788574594.369817:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574594.369823:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574594.369825:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574594.369828:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369830:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.369832:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369833:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574594.369836:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574594.369838:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574594.369840:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574594.369842:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574594.369859:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574594.369861:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574594.369863:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369865:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.369866:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369868:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574594.369870:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574594.369872:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369874:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574594.369876:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574594.369879:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574594.369881:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.369883:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574594.369885:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574594.369890:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574594.369893:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574594.369896:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574594.369897:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.369899:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574594.369902:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574594.369905:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574594.369912:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574594.369918:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.369925:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810289792/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574594.369932:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574594.369936:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.369939:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810289792/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574594.369958:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574594.369961:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574594.369969:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574594.369970:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574594.369972:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574594.369973:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574594.369975:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15060 00000400:00000200:0.0:1788574594.369978:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574594.369981:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574594.369983:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574594.369985:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369987:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.369989:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.369991:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574594.369992:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574594.369993:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574594.369994:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574594.369996:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574594.369998:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574594.370003:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574594.370006:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370009:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370011:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370015:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574594.370017:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574594.370021:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370023:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574594.370025:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574594.370031:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574594.370035:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370041:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574594.370045:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574594.370053:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370058:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574594.370065:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574594.370068:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574594.370072:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574594.370073:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370074:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574594.370075:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574594.370078:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574594.370084:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574594.370090:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.370097:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810289920/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574594.370110:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574594.370113:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.370118:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810289920/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574594.370126:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574594.370129:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574594.370132:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574594.370135:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574594.370137:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574594.370140:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574594.370144:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15060 00000400:00000200:0.0:1788574594.370150:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574594.370154:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574594.370156:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574594.370161:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370165:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370169:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370178:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574594.370183:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574594.370187:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574594.370191:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574594.370197:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574594.370202:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574594.370206:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574594.370214:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370220:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370226:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370244:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574594.370248:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574594.370251:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370254:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574594.370256:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574594.370261:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574594.370265:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370269:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574594.370272:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574594.370276:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370278:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574594.370283:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574594.370289:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574594.370293:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574594.370295:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370299:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574594.370303:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574594.370307:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574594.370324:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574594.370335:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.370345:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810290048/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574594.370375:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574594.370379:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.370386:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810290048/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574594.370404:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574594.370408:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574594.370416:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574594.370420:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574594.370422:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574594.370425:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574594.370430:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15060 00000400:00000200:0.0:1788574594.370438:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574594.370443:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574594.370447:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574594.370454:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370458:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370463:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370468:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574594.370471:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574594.370472:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574594.370473:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574594.370476:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574594.370479:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574594.370484:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574594.370490:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370493:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370496:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370499:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574594.370508:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574594.370512:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370515:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574594.370518:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574594.370521:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574594.370525:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370527:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574594.370529:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574594.370533:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370535:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574594.370542:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574594.370546:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574594.370550:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574594.370552:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370554:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574594.370555:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574594.370558:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574594.370564:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574594.370569:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.370577:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810290176/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574594.370585:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574594.370587:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.370593:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810290176/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574594.370605:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574594.370609:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574594.370616:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574594.370619:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574594.370621:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574594.370625:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574594.370688:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15060 00000400:00000200:0.0:1788574594.370697:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574594.370703:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574594.370706:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574594.370713:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370718:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370723:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370727:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574594.370732:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574594.370733:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574594.370735:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574594.370737:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574594.370738:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574594.370743:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574594.370750:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370753:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574594.370758:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370761:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574594.370763:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574594.370770:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574594.370775:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574594.370782:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574594.370788:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574594.370793:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370797:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574594.370802:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574594.370808:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370813:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574594.370820:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574594.370826:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574594.370833:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574594.370844:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.370850:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574594.370855:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574594.370861:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574594.370867:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574594.370874:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.370885:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810290304/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574594.370894:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574594.370897:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.370903:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810290304/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574594.370917:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574594.370923:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15071) 00000100:00080000:0.0:1788574595.458349:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574595.458355:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574595.458357:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574595.458359:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574595.458362:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574595.458366:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15066 00000400:00000200:0.0:1788574595.458373:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574595.458378:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574595.458381:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574595.458384:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458387:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458390:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458392:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574595.458395:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574595.458397:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574595.458399:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574595.458402:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574595.458404:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574595.458416:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574595.458420:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458423:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458425:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458428:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574595.458431:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574595.458434:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458438:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574595.458440:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574595.458443:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574595.458445:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458447:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574595.458449:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574595.458452:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574595.458454:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574595.458457:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574595.458458:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458460:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574595.458461:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574595.458463:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574595.458468:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574595.458471:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574595.458476:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810290560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574595.458482:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574595.458484:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.458488:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810290560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574595.458506:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574595.458509:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574595.458513:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574595.458528:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574595.458530:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574595.458533:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574595.458536:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15066 00000400:00000200:0.0:1788574595.458541:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574595.458544:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574595.458547:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574595.458550:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458552:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458554:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458556:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574595.458559:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574595.458560:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574595.458562:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574595.458565:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574595.458567:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574595.458570:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574595.458573:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458575:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458577:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458579:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574595.458581:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574595.458584:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458587:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574595.458589:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574595.458594:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574595.458597:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458599:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574595.458603:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574595.458606:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458613:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574595.458616:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574595.458619:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574595.458621:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574595.458623:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458625:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574595.458626:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574595.458628:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574595.458632:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574595.458670:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574595.458676:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810290688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574595.458685:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574595.458688:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.458693:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810290688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574595.458703:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574595.458706:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574595.458709:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574595.458711:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574595.458712:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574595.458715:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574595.458718:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15066 00000400:00000200:0.0:1788574595.458756:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574595.458759:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574595.458762:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574595.458765:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458766:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458769:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458771:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574595.458780:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574595.458786:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574595.458788:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574595.458790:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574595.458792:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574595.458794:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574595.458798:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458800:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458802:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458804:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574595.458806:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574595.458809:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458812:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574595.458814:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574595.458817:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574595.458819:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458821:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574595.458824:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574595.458826:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458828:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574595.458832:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574595.458835:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574595.458838:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574595.458839:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458841:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574595.458843:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574595.458845:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574595.458849:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574595.458852:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574595.458856:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810290816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574595.458864:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574595.458866:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.458875:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810290816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574595.458883:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574595.458886:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574595.458890:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574595.458891:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574595.458893:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574595.458896:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574595.458898:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15066 00000400:00000200:0.0:1788574595.458902:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574595.458905:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574595.458907:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574595.458910:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458912:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458922:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458924:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574595.458930:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574595.458931:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574595.458933:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574595.458935:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574595.458938:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574595.458940:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574595.458943:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458945:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.458948:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458950:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574595.458953:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574595.458955:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574595.458957:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574595.458963:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574595.458966:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574595.458968:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458971:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574595.458973:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574595.458975:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458977:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574595.458981:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574595.458984:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574595.458986:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574595.458988:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.458989:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574595.458991:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574595.458993:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574595.458997:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574595.459001:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574595.459006:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810290944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574595.459014:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574595.459016:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.459020:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810290944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574595.459028:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574595.459031:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574595.459035:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574595.459036:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574595.459038:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574595.459041:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574595.459044:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15066 00000400:00000200:0.0:1788574595.459049:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574595.459052:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574595.459055:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574595.459064:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.459066:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.459068:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.459070:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574595.459073:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574595.459074:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574595.459076:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574595.459078:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574595.459080:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574595.459082:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574595.459086:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.459088:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574595.459090:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574595.459093:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574595.459095:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574595.459098:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574595.459100:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574595.459102:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574595.459105:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574595.459107:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.459109:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574595.459112:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574595.459114:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.459116:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574595.459119:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574595.459121:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574595.459124:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574595.459125:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574595.459127:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574595.459128:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574595.459130:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574595.459134:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574595.459143:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574595.459147:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810291072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574595.459155:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574595.459158:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.459161:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810291072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574595.459170:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574595.459174:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15072) 00000020:00000001:0.0:1788574597.313863:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574597.313870:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574597.313875:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574598.337869:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574598.337875:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574598.337879:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574599.297778:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309738271, at 4309738304 00000100:00000010:0.0:1788574599.297805:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788574602.433812:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574602.433815:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574602.433818:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574603.457826:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574603.457830:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574603.457832:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574604.353735:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309743327, at 4309743360 00000100:00000010:0.0:1788574604.353748:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574607.553812:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574607.553815:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574607.553818:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574608.577796:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574608.577801:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574608.577805:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788574608.705904:0:2393:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788574608.705954:0:2393:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574608.705976:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574608.705980:0:2393:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788574608.705985:0:2393:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574608.705989:0:2393:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574608.705994:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574608.705999:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9b00. 00000400:00000200:0.0:1788574608.706005:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574608.706010:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574608.706018:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3550 00000400:00000010:0.0:1788574608.706020:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3550. 00000100:00000001:0.0:1788574608.706025:0:2393:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574608.706032:0:2393:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6bb80 x1875440810291840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706052:0:2393:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574608.706055:0:2393:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6bb80 x1875440810291840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706062:0:2393:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574608.706063:0:2393:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574608.706072:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574608.706075:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9200. 00000400:00000200:0.0:1788574608.706078:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574608.706087:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574608.706091:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3440 00000400:00000010:0.0:1788574608.706093:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3440. 00000100:00000001:0.0:1788574608.706097:0:2393:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574608.706101:0:2393:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68000 x1875440810291968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706110:0:2393:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574608.706116:0:2393:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68000 x1875440810291968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706127:0:2393:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574608.706139:0:2393:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574608.706166:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574608.706169:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9700. 00000400:00000200:0.0:1788574608.706173:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574608.706176:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574608.706179:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3a18 00000400:00000010:0.0:1788574608.706186:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3a18. 00000100:00000001:0.0:1788574608.706190:0:2393:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574608.706197:0:2393:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69c00 x1875440810292096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706221:0:2393:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574608.706224:0:2393:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69c00 x1875440810292096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706235:0:2393:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574608.706240:0:2393:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574608.706250:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574608.706255:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9d00. 00000400:00000200:0.0:1788574608.706260:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574608.706269:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574608.706276:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3990 00000400:00000010:0.0:1788574608.706280:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3990. 00000100:00000001:0.0:1788574608.706285:0:2393:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574608.706292:0:2393:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69f80 x1875440810292224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706312:0:2393:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574608.706319:0:2393:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69f80 x1875440810292224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706332:0:2393:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574608.706337:0:2393:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574608.706347:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788574608.706351:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c37042e9000. 00000400:00000200:0.0:1788574608.706357:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574608.706371:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574608.706380:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37319f3c38 00000400:00000010:0.0:1788574608.706384:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37319f3c38. 00000100:00000001:0.0:1788574608.706390:0:2393:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574608.706398:0:2393:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed69880 x1875440810292352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706416:0:2393:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574608.706423:0:2393:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed69880 x1875440810292352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574608.706430:0:2393:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574608.706432:0:2393:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788574609.473733:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309748446, at 4309748480 00000100:00000010:0.0:1788574609.473752:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788574612.673813:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574612.673817:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574612.673820:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574613.697792:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574613.697795:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574613.697798:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574614.593719:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309753566, at 4309753600 00000100:00000010:0.0:1788574614.593744:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788574617.793898:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574617.793902:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574617.793904:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574618.817849:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574618.817854:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574618.817857:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574621.505738:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309760479, at 4309760512 00000100:00000010:0.0:1788574621.505753:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788574622.913840:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574622.913844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574622.913847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574623.937826:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574623.937843:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574623.937846:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574626.881753:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309765854, at 4309765888 00000100:00000010:0.0:1788574626.881775:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788574628.033840:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574628.033844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574628.033846:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574629.057850:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574629.057856:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574629.057861:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574632.001729:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309770974, at 4309771008 00000100:00000010:0.0:1788574632.001742:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574633.153811:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574633.153815:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574633.153818:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574634.177870:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574634.177876:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574634.177879:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574637.121747:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309776095, at 4309776128 00000100:00000010:0.0:1788574637.121761:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788574638.273840:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574638.273844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574638.273847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574639.297861:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574639.297865:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574639.297868:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574641.409947:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574641.409953:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574641.409963:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574641.409967:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574641.409969:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574642.945749:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309781919, at 4309781952 00000100:00000010:0.0:1788574642.945765:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788574643.393822:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574643.393865:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574643.393868:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574644.417807:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574644.417810:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574644.417813:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574648.385749:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309787358, at 4309787392 00000100:00000010:0.0:1788574648.385765:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788574648.513800:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574648.513804:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574648.513807:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574649.537874:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574649.537878:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574649.537881:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574653.505715:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309792478, at 4309792512 00000100:00000010:0.0:1788574653.505729:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788574653.633859:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574653.633862:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574653.633865:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574654.657830:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574654.657834:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574654.657837:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574658.625749:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309797598, at 4309797632 00000100:00000010:0.0:1788574658.625763:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215f00. 00000020:00000001:0.0:1788574658.753817:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574658.753821:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574658.753824:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574659.777940:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574659.777947:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574659.777952:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574659.905799:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574659.905808:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574659.905810:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574659.905814:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574659.905817:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574659.905837:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15126 00000400:00000200:0.0:1788574659.905845:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574659.905850:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574659.905852:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574659.905855:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905858:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.905860:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905861:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574659.905864:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574659.905865:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574659.905868:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574659.905870:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574659.905872:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574659.905874:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574659.905876:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905877:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.905878:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905880:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574659.905882:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574659.905885:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905886:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574659.905889:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574659.905891:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574659.905893:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.905895:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574659.905897:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574659.905901:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574659.905904:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574659.905907:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574659.905909:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.905911:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574659.905914:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574659.905920:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574659.905927:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574659.905932:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.905939:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810299264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574659.905947:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574659.905950:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.905953:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810299264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574659.905974:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574659.905976:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574659.905979:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574659.905980:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574659.905982:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574659.905983:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574659.905985:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15126 00000400:00000200:0.0:1788574659.905988:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574659.905990:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574659.905992:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574659.905994:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905996:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.905997:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.905999:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574659.906000:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574659.906001:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574659.906003:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574659.906004:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574659.906006:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574659.906008:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574659.906011:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906016:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906018:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906020:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574659.906022:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574659.906029:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906035:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574659.906039:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574659.906047:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574659.906052:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906055:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574659.906060:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574659.906067:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906081:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574659.906086:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574659.906091:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574659.906094:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574659.906098:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906104:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574659.906105:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574659.906107:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574659.906114:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574659.906118:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.906124:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810299392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574659.906137:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574659.906140:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.906147:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810299392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574659.906167:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574659.906171:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574659.906177:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574659.906187:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574659.906190:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574659.906192:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574659.906196:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15126 00000400:00000200:0.0:1788574659.906201:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574659.906206:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574659.906209:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574659.906213:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906217:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906229:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906234:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574659.906239:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574659.906243:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574659.906246:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574659.906248:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574659.906253:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574659.906257:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574659.906261:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906262:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906266:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906271:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574659.906272:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574659.906279:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906282:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574659.906284:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574659.906292:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574659.906297:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906299:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574659.906303:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574659.906308:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906320:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574659.906324:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574659.906327:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574659.906332:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574659.906336:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906341:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574659.906342:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574659.906345:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574659.906349:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574659.906356:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.906365:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810299520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574659.906373:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574659.906376:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.906385:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810299520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574659.906398:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574659.906401:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574659.906406:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574659.906409:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574659.906412:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574659.906416:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574659.906420:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15126 00000400:00000200:0.0:1788574659.906424:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574659.906429:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574659.906432:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574659.906435:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906437:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906441:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906445:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574659.906448:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574659.906455:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574659.906459:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574659.906462:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574659.906467:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574659.906471:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574659.906477:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906478:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906480:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906484:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574659.906488:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574659.906493:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906499:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574659.906501:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574659.906503:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574659.906509:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906512:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574659.906516:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574659.906520:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906523:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574659.906529:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574659.906533:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574659.906538:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574659.906540:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906544:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574659.906548:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574659.906551:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574659.906557:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574659.906562:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.906570:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810299648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574659.906590:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574659.906593:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.906608:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810299648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574659.906627:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574659.906630:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574659.906666:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574659.906684:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574659.906687:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574659.906693:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574659.906698:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15126 00000400:00000200:0.0:1788574659.906705:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574659.906712:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574659.906714:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574659.906719:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906722:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906728:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906732:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574659.906737:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574659.906740:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574659.906742:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574659.906745:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574659.906746:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574659.906747:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574659.906749:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906752:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574659.906754:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906757:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574659.906759:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574659.906761:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574659.906763:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574659.906771:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574659.906775:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574659.906779:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906780:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574659.906783:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574659.906787:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906791:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574659.906794:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574659.906798:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574659.906802:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574659.906804:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574659.906808:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574659.906809:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574659.906813:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574659.906817:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574659.906823:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.906830:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810299776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574659.906843:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574659.906847:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.906854:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810299776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574659.906870:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574659.906875:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15136) 00000100:00080000:0.0:1788574660.610366:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574660.610375:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574660.610378:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574660.610382:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574660.610387:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574660.610393:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15131 00000400:00000200:0.0:1788574660.610403:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574660.610408:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574660.610427:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574660.610430:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610433:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610436:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610438:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574660.610441:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574660.610443:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574660.610445:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574660.610448:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574660.610450:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574660.610452:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574660.610471:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610473:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610474:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610476:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574660.610478:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574660.610481:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610483:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574660.610485:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574660.610488:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574660.610490:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.610492:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574660.610493:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574660.610498:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574660.610500:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574660.610503:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574660.610504:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.610507:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574660.610509:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574660.610512:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574660.610518:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574660.610523:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574660.610562:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810300032/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574660.610572:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574660.610575:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.610580:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810300032/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574660.610603:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574660.610606:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574660.610613:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574660.610614:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574660.610616:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574660.610618:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574660.610620:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15131 00000400:00000200:0.0:1788574660.610624:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574660.610627:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574660.610629:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574660.610632:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610634:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610691:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610711:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574660.610716:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574660.610719:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574660.610723:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574660.610727:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574660.610732:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574660.610738:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574660.610753:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610756:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610761:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610771:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574660.610776:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574660.610782:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610784:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574660.610786:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574660.610794:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574660.610799:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.610802:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574660.610807:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574660.610810:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.610812:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574660.610821:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574660.610827:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574660.610832:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574660.610835:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.610838:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574660.610840:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574660.610843:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574660.610850:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574660.610856:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574660.610865:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810300160/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574660.610877:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574660.610882:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.610886:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810300160/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574660.610900:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574660.610904:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574660.610909:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574660.610912:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574660.610914:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574660.610926:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574660.610930:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15131 00000400:00000200:0.0:1788574660.610938:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574660.610944:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574660.610949:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574660.610957:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610962:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610964:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610965:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574660.610967:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574660.610970:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574660.610973:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574660.610977:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574660.610982:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574660.610984:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574660.610991:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.610994:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.610996:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611001:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574660.611003:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574660.611009:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611014:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574660.611016:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574660.611021:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574660.611027:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611031:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574660.611034:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574660.611039:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611041:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574660.611046:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574660.611053:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574660.611059:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574660.611063:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611069:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574660.611073:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574660.611078:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574660.611086:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574660.611094:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574660.611110:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810300288/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574660.611137:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574660.611141:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.611149:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810300288/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574660.611159:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574660.611161:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574660.611166:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574660.611169:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574660.611171:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574660.611176:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574660.611179:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15131 00000400:00000200:0.0:1788574660.611184:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574660.611195:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574660.611198:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574660.611203:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611207:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.611213:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611215:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574660.611221:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574660.611224:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574660.611233:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574660.611237:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574660.611242:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574660.611247:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574660.611263:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611267:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.611271:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611276:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574660.611279:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574660.611284:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611290:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574660.611293:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574660.611298:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574660.611304:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611307:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574660.611310:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574660.611312:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611313:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574660.611318:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574660.611321:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574660.611333:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574660.611336:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611339:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574660.611342:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574660.611345:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574660.611351:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574660.611357:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574660.611365:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810300416/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574660.611382:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574660.611387:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.611391:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810300416/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574660.611419:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574660.611423:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574660.611425:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574660.611426:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574660.611428:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574660.611430:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574660.611434:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15131 00000400:00000200:0.0:1788574660.611442:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574660.611447:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574660.611451:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574660.611458:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611462:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.611468:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611471:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574660.611473:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574660.611476:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574660.611479:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574660.611481:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574660.611484:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574660.611489:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574660.611497:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611500:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574660.611504:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611508:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574660.611509:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574660.611511:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574660.611513:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574660.611520:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574660.611524:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574660.611530:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611533:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574660.611537:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574660.611541:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611544:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574660.611548:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574660.611553:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574660.611558:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574660.611560:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574660.611561:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574660.611562:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574660.611564:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574660.611570:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574660.611576:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574660.611582:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810300544/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574660.611600:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574660.611604:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.611608:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810300544/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574660.611622:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574660.611629:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15137) 00000020:00000001:0.0:1788574663.873881:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574663.873886:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574663.873890:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574664.449735:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309803423, at 4309803456 00000100:00000010:0.0:1788574664.449755:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215540. 00000020:00000001:0.0:1788574664.897823:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574664.897827:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574664.897831:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574666.049756:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574666.049789:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574666.049792:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574666.049796:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574666.049801:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574666.049808:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15132 00000400:00000200:0.0:1788574666.049819:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574666.049826:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574666.049829:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574666.049831:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049835:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.049837:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049839:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574666.049843:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574666.049845:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574666.049848:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574666.049851:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574666.049853:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574666.049854:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574666.049857:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049858:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.049860:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049861:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574666.049864:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574666.049867:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049868:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574666.049871:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574666.049874:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574666.049877:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.049878:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574666.049880:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574666.049887:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574666.049894:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574666.049899:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574666.049900:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.049903:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574666.049906:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574666.049911:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574666.049918:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574666.049926:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.049936:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810300672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574666.049945:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574666.049950:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.049954:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810300672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574666.049975:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574666.049978:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574666.049981:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574666.049982:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574666.049984:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574666.049986:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574666.049988:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15132 00000400:00000200:0.0:1788574666.049990:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574666.049992:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574666.049994:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574666.049996:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.049997:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.049999:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050001:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574666.050003:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574666.050004:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574666.050006:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574666.050012:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574666.050014:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574666.050024:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574666.050029:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050032:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050036:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050037:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574666.050040:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574666.050044:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050049:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574666.050052:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574666.050061:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574666.050064:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050066:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574666.050075:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574666.050081:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050084:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574666.050091:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574666.050098:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574666.050104:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574666.050107:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050111:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574666.050114:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574666.050120:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574666.050128:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574666.050136:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.050147:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810300800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574666.050161:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574666.050164:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.050168:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810300800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574666.050216:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574666.050218:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574666.050221:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574666.050223:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574666.050225:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574666.050230:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574666.050235:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15132 00000400:00000200:0.0:1788574666.050241:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574666.050246:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574666.050250:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574666.050254:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050259:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050264:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050265:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574666.050267:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574666.050268:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574666.050269:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574666.050273:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574666.050277:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574666.050278:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574666.050284:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050297:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050304:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050308:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574666.050312:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574666.050317:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050320:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574666.050327:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574666.050331:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574666.050336:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050339:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574666.050342:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574666.050345:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050348:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574666.050357:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574666.050360:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574666.050366:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574666.050369:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050375:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574666.050377:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574666.050381:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574666.050390:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574666.050397:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.050407:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810300928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574666.050423:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574666.050428:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.050437:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810300928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574666.050457:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574666.050463:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574666.050471:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574666.050474:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574666.050477:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574666.050483:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574666.050489:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15132 00000400:00000200:0.0:1788574666.050496:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574666.050502:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574666.050506:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574666.050518:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050523:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050527:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050532:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574666.050536:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574666.050537:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574666.050538:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574666.050542:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574666.050546:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574666.050550:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574666.050556:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050570:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050573:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050577:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574666.050582:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574666.050587:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050592:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574666.050594:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574666.050597:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574666.050600:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050602:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574666.050604:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574666.050622:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050624:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574666.050627:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574666.050630:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574666.050663:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574666.050665:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050668:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574666.050670:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574666.050674:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574666.050679:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574666.050689:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.050696:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810301056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574666.050719:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574666.050723:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.050733:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810301056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574666.050753:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574666.050760:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574666.050770:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574666.050773:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574666.050778:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574666.050782:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574666.050785:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15132 00000400:00000200:0.0:1788574666.050793:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574666.050801:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574666.050805:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574666.050812:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050818:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050823:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050829:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574666.050836:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574666.050840:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574666.050845:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574666.050850:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574666.050856:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574666.050864:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574666.050874:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050878:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574666.050888:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050890:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574666.050892:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574666.050896:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574666.050901:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574666.050904:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574666.050907:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574666.050913:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050917:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574666.050921:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574666.050927:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050930:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574666.050939:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574666.050944:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574666.050950:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574666.050952:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.050955:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574666.050957:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574666.050961:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574666.050967:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574666.050973:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.050978:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810301184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574666.050990:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574666.050993:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.050997:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810301184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574666.051009:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574666.051015:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15142) 00000020:00000001:0.0:1788574668.993891:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574668.993899:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574668.993904:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574669.889742:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309808862, at 4309808896 00000100:00000010:0.0:1788574669.889761:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215a80. 00000020:00000001:0.0:1788574670.017852:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574670.017857:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574670.017861:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574674.113844:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574674.113851:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574674.113854:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574675.009724:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309813982, at 4309814016 00000100:00000010:0.0:1788574675.009738:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215480. 00000020:00000001:0.0:1788574675.137911:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574675.137922:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574675.137926:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574679.233815:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574679.233819:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574679.233822:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574679.297955:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574679.297963:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574679.297986:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574679.297995:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574679.298000:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788574680.257777:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574680.257781:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574680.257783:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788574680.641727:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574680.641741:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9940. 00000020:00000010:0.0:1788574680.641749:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574680.641756:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574680.641760:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574680.641765:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574680.641770:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788574680.641775:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574680.641779:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574680.641783:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ac00. 10000000:00000040:0.0:1788574680.641787:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574680.641799:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574680.641804:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788574680.641809:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b400. 00000020:00000010:0.0:1788574680.641810:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574680.641813:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788574680.641815:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116265a0. 00040000:00000010:0.0:1788574680.641817:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574680.641819:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ac00. 00000020:00000010:0.0:1788574680.641821:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788574680.641822:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9940. 10000000:01000000:0.0:1788574680.641824:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574680.641827:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574680.641828:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574680.641830:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574680.641831:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574680.641833:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574680.641834:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574680.641836:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574680.641837:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574680.641839:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574680.641841:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574680.641842:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574680.641843:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574681.985730:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574681.985736:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574681.985740:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574681.985742:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574681.985743:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574681.985744:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574681.985746:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574681.985748:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574681.985750:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574681.985752:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574681.985753:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574681.985768:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574681.985773:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574681.985791:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574681.985794:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574681.985800:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574681.985814:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788574681.985820:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574681.985823:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574681.985826:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574681.985833:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788574681.985837:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574681.985840:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788574681.985845:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574681.985848:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168c6b to hash 00000020:00000001:0.0:1788574681.985850:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574681.985851:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574681.985854:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574681.985857:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574681.985861:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574681.985865:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574681.985874:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574681.985882:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574681.985887:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574681.985888:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574681.985893:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61e40. 00000100:00000001:0.0:1788574681.985895:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612352 : -109714523939264 : ffff9c3718c61e40) 00000100:00000001:0.0:1788574681.985897:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574681.985899:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788574681.985903:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574681.985907:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574681.985916:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574681.985918:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574681.985927:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574681.985937:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574681.985941:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574681.985947:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574681.985950:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574681.985954:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574681.985956:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574681.985958:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574681.985959:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61e40 going to sleep for 0 seconds 00000100:00000001:0.0:1788574681.985961:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574681.985962:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574681.985971:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574681.985976:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574681.985979:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574681.985982:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574681.985989:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810303232:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574681.985994:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574681.985997:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574681.985999:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574681.986001:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574681.986004:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574681.986010:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574681.986013:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61e40. 00000100:00000001:0.0:1788574681.986024:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574681.986025:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574681.986028:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574681.986031:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574681.986032:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574681.986034:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788574681.986036:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574681.986038:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788574681.986042:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986048:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574681.986052:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986057:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574681.986060:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986066:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574681.986068:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574681.986070:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986076:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574681.986077:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574681.986080:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574681.986082:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574681.986089:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574681.986091:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574681.986099:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986105:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574681.986106:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574681.986107:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574681.986109:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788574681.986111:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574681.986120:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788574681.986123:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574681.986125:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574681.986129:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574681.986130:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574681.986134:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574681.986135:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574681.986136:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574681.986137:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574681.986139:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574681.986140:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574681.986141:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574681.986142:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574681.986143:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574681.986145:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168c6b from hash 00010000:00000001:0.0:1788574681.986147:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574681.986148:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574681.986148:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574681.986149:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574681.986151:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574681.986152:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574681.986153:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574681.986154:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574681.986158:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574681.986159:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574681.986161:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574681.986165:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574681.986165:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574681.986166:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574681.986171:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574681.986172:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574681.986172:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574681.986173:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574681.986174:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168c6b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574681.986178:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788574681.986185:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574681.986186:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574681.986188:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574681.986191:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810303232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574681.986196:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574681.986198:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574681.986199:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574681.986200:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574681.986205:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788574681.986209:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574681.986210:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574681.986211:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788574681.986213:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574681.986214:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574681.986215:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574681.986217:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574681.986219:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574684.353809:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574684.353812:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574684.353821:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574685.377799:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574685.377803:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574685.377806:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574685.825728:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309824799, at 4309824832 00000100:00000010:0.0:1788574685.825745:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215360. 00000020:00000001:0.0:1788574689.473852:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574689.473860:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574689.473866:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574690.497841:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574690.497844:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574690.497847:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574690.881753:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309829854, at 4309829888 00000100:00000010:0.0:1788574690.881785:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00000001:0.0:1788574692.162105:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574692.162112:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574692.162118:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574692.162138:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.162159:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.162163:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.162171:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574692.162181:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574692.162191:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.162196:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810304640:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574692.162202:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574692.162207:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574692.162211:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.162215:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574692.162222:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788574692.162230:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.162241:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574692.162247:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574692.162261:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574692.162279:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810304640, portal 25 00000100:00000001:0.0:1788574692.162283:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574692.162287:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788574692.162298:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.162309:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574692.162314:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574692.162317:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574692.162321:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810304640, offset 0 00000400:00000200:0.0:1788574692.162330:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574692.162340:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.162354:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574692.162368:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574692.162374:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574692.162382:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574692.162387:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574692.162396:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542427:542427:256:4294967295] 192.168.202.148@tcp LPNI seq info [542392:542392:8:4294967295] 00000400:00000200:0.0:1788574692.162428:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.162459:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574692.162476:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.163699:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574692.163710:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574692.163713:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574692.163728:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.163731:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574692.163750:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574692.163757:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574692.163765:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574692.163768:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574692.163774:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574692.163780:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163802:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.163807:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163816:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.163818:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574692.163827:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.163833:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574692.163836:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4309834976 00000100:00000010:0.0:1788574692.163841:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788574692.163848:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574692.163854:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4309834976 00000100:00000010:0.0:1788574692.163856:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626c00. 00000100:00000001:0.0:1788574692.163859:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574692.163862:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574692.163864:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.163866:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574692.163870:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574692/real 1788574692] req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574692.163879:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574692.163880:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574692.163883:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574692.163886:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574692.163887:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574692.163889:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574692.163892:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574692.163898:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163902:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574692.163908:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.163910:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.163912:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574692.163915:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163921:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574692.163925:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163931:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574692.163933:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.163934:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.163935:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.163937:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574692.163939:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574692.163941:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574692.163945:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574692.163958:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574692.163964:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574692.163973:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810304640:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574692.163979:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.163982:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.163987:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810304640/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.163995:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574692.163999:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574692.164003:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788574692.164006:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574692.164008:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574692.164014:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574692.164016:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574692.164019:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e800. 02000000:00000001:0.0:1788574692.164022:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574692.164024:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574692.164026:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788574692.164029:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574692.164030:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164032:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164040:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574692.164042:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164044:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574692.164046:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574692.164047:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574692.164051:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164057:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.164058:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164060:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574692.164062:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574692.164064:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.164065:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810304768:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574692.164068:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574692.164070:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574692.164071:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164073:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574692.164075:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788574692.164077:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.164080:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574692.164082:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574692.164086:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574692.164089:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810304768, portal 10 00000100:00000001:0.0:1788574692.164090:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574692.164091:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788574692.164097:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164106:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574692.164107:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574692.164109:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574692.164111:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810304768, offset 0 00000400:00000200:0.0:1788574692.164114:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164117:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164120:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574692.164123:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574692.164125:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574692.164127:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574692.164129:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164132:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542428:542428:256:4294967295] 192.168.202.148@tcp LPNI seq info [542393:542393:8:4294967295] 00000400:00000200:0.0:1788574692.164135:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164140:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574692.164144:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164146:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574692.164150:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574692.164152:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574692.164160:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164161:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574692.164163:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574692.164166:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574692.164168:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574692.164170:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574692.164171:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574692.164174:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164180:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.164182:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164195:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164196:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574692.164199:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574692.164203:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788574692.164205:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574692.164207:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309831194 00000100:00000001:0.0:1788574692.164209:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164211:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309831194 00000100:00000001:0.0:1788574692.164212:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164214:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574692.164215:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164216:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574692.164219:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574692/real 1788574692] req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574692.164224:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574692.164225:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574692.164227:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574692.164228:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574692.164229:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574692.164231:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574692.164233:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574692.164238:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164243:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574692.164244:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164245:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164246:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574692.164248:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164256:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574692.164259:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164264:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574692.164265:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164266:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164267:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164268:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574692.164269:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574692.164271:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574692.164274:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574692.164275:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574692.164278:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574692.164283:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810304768:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574692.164286:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574692.164288:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164292:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.164293:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164295:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574692.164296:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574692.164297:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.164299:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810304896:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574692.164301:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574692.164302:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574692.164306:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164308:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574692.164310:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574692.164311:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.164314:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574692.164316:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574692.164318:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574692.164320:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810304896, portal 10 00000100:00000001:0.0:1788574692.164322:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574692.164323:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788574692.164328:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164332:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574692.164334:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574692.164335:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574692.164337:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810304896, offset 0 00000400:00000200:0.0:1788574692.164339:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164342:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164344:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574692.164346:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574692.164347:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574692.164349:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574692.164351:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164353:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542429:542429:256:4294967295] 192.168.202.148@tcp LPNI seq info [542394:542394:8:4294967295] 00000400:00000200:0.0:1788574692.164356:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164360:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574692.164363:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164365:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574692.164367:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574692.164369:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574692.164382:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164386:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574692.164388:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574692.164391:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574692.164393:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574692.164394:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574692.164396:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574692.164399:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164405:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.164408:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164415:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164417:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574692.164420:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574692.164425:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788574692.164428:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574692.164430:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309831198 00000100:00000001:0.0:1788574692.164433:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164435:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309831198 00000100:00000001:0.0:1788574692.164438:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164440:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574692.164442:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164444:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574692.164449:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574692/real 1788574692] req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574692.164458:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574692.164459:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574692.164461:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574692.164463:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574692.164465:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574692.164471:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574692.164474:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574692.164480:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164485:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574692.164486:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164487:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164488:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574692.164490:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164496:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574692.164500:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164508:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574692.164511:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164512:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164514:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164516:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574692.164518:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574692.164521:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574692.164525:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574692.164528:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574692.164533:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574692.164542:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810304896:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574692.164546:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574692.164554:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164559:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.164561:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164563:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574692.164564:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574692.164566:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.164568:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810305024:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574692.164570:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574692.164572:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574692.164573:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164574:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574692.164577:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574692.164579:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.164582:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574692.164584:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574692.164587:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574692.164591:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810305024, portal 4 00000100:00000001:0.0:1788574692.164593:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574692.164594:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788574692.164601:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164608:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574692.164611:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574692.164613:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574692.164616:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810305024, offset 0 00000400:00000200:0.0:1788574692.164621:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164625:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164629:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574692.164632:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574692.164664:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574692.164667:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574692.164669:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164677:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542430:542430:256:4294967295] 192.168.202.148@tcp LPNI seq info [542395:542395:8:4294967295] 00000400:00000200:0.0:1788574692.164681:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164686:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574692.164689:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164692:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574692.164695:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574692.164697:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574692.164705:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164707:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574692.164709:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574692.164712:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574692.164714:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574692.164715:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574692.164718:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574692.164722:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164730:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.164733:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164740:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164742:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574692.164745:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574692.164749:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788574692.164752:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574692.164755:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309831331 00000100:00000001:0.0:1788574692.164758:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164760:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309831331 00000100:00000001:0.0:1788574692.164762:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.164765:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574692.164772:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.164774:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574692.164777:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574692/real 1788574692] req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574692.164784:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574692.164786:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574692.164788:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574692.164791:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574692.164793:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574692.164795:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574692.164798:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574692.164808:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164816:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574692.164817:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164819:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164821:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574692.164825:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164833:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574692.164837:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164842:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574692.164844:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164845:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.164846:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.164847:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574692.164850:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574692.164852:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574692.164859:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574692.164860:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574692.164864:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574692.164869:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810305024:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574692.164872:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574692.164874:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164878:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.164879:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164881:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574692.164883:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574692.164884:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.164886:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810305152:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574692.164888:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574692.164890:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574692.164891:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574692.164892:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574692.164894:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788574692.164896:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574692.164898:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574692.164900:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574692.164903:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574692.164905:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810305152, portal 4 00000100:00000001:0.0:1788574692.164907:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574692.164908:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286346880 : -109714423204736 : ffff9c371ec73480) 00000100:00000040:0.0:1788574692.164912:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.164917:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574692.164923:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574692.164925:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574692.164927:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810305152, offset 0 00000400:00000200:0.0:1788574692.164930:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164933:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164935:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574692.164938:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574692.164939:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574692.164941:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574692.164943:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574692.164946:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542431:542431:256:4294967295] 192.168.202.148@tcp LPNI seq info [542396:542396:8:4294967295] 00000400:00000200:0.0:1788574692.164950:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574692.164957:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574692.164962:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164965:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574692.164969:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574692.164973:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574692.164984:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574692.164987:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574692.164990:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574692.164994:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574692.164998:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574692.165000:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574692.165002:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574692.165005:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165011:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.165013:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165017:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.165019:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574692.165026:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574692.165029:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788574692.165032:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574692.165033:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309831340 00000100:00000001:0.0:1788574692.165035:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.165037:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309831340 00000100:00000001:0.0:1788574692.165039:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309834976 00000100:00000001:0.0:1788574692.165049:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574692.165050:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.165052:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574692.165054:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574692/real 1788574692] req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574692.165063:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574692.165064:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574692.165066:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574692.165068:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574692.165069:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574692.165070:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574692.165073:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574692.165078:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165083:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574692.165084:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165085:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.165086:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574692.165088:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165092:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574692.165094:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165103:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574692.165105:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165106:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574692.165107:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165108:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574692.165109:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574692.165111:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574692.165113:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574692.165115:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574692.165117:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574692.165123:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810305152:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574692.165125:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165126:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.165128:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810304768/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165132:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574692.165133:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574692.165135:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20dc00. 02000000:00000001:0.0:1788574692.165136:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574692.165138:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574692.165139:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574692.165142:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574692.165144:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f000. 02000000:00000001:0.0:1788574692.165146:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574692.165148:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574692.165149:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788574692.165152:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574692.165154:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165161:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.165165:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810304896/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165173:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574692.165175:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574692.165177:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574692.165179:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574692.165181:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574692.165182:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574692.165185:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574692.165186:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c800. 02000000:00000001:0.0:1788574692.165189:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574692.165190:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574692.165191:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788574692.165193:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574692.165194:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165195:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.165198:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810305024/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165202:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574692.165203:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574692.165204:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574692.165206:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574692.165207:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574692.165208:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788574692.165209:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574692.165210:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788574692.165211:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574692.165211:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574692.165212:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788574692.165213:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574692.165214:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165215:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574692.165217:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec73480 x1875440810305152/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574692.165229:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574692.165230:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574692.165231:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d400. 02000000:00000001:0.0:1788574692.165233:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574692.165234:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574692.165234:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788574692.165235:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574692.165236:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d800. 02000000:00000001:0.0:1788574692.165238:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574692.165238:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574692.165239:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec73480. 00000100:00000001:0.0:1788574692.165241:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574692.165242:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165243:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.165246:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574692.165246:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574692.165248:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574692.165249:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574694.593814:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574694.593820:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574694.593824:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574695.617793:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574695.617798:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574695.617802:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574696.001716:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309834976, at 4309835008 00000100:00000010:0.0:1788574696.001733:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000100:00080000:0.0:1788574697.281722:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574697.281731:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574697.281733:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574697.281736:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574697.281740:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574697.281746:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15163 00000400:00000200:0.0:1788574697.281755:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574697.281778:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574697.281782:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574697.281786:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281789:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.281791:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281795:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574697.281799:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574697.281801:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574697.281805:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574697.281808:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574697.281811:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574697.281814:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574697.281817:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281819:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.281821:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281823:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574697.281827:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574697.281830:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281833:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574697.281836:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574697.281839:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574697.281841:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.281843:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574697.281844:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574697.281851:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574697.281853:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574697.281857:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574697.281858:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.281860:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574697.281863:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574697.281866:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574697.281872:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574697.281882:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.281889:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810305280/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574697.281897:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574697.281900:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.281903:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810305280/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574697.281924:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574697.281928:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574697.281931:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574697.281933:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574697.281935:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574697.281937:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574697.281948:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15163 00000400:00000200:0.0:1788574697.281954:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574697.281958:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574697.281960:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574697.281963:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281965:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.281967:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281970:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574697.281972:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574697.281973:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574697.281975:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574697.281978:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574697.281980:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574697.281982:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574697.281985:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281988:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.281996:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.281998:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574697.282001:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574697.282004:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282006:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574697.282008:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574697.282014:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574697.282016:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282018:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574697.282022:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574697.282026:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282028:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574697.282031:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574697.282034:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574697.282036:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574697.282038:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282040:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574697.282042:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574697.282044:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574697.282047:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574697.282050:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.282055:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810305408/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574697.282063:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574697.282066:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.282069:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810305408/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574697.282079:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574697.282090:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574697.282093:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574697.282095:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574697.282101:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574697.282104:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574697.282107:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15163 00000400:00000200:0.0:1788574697.282147:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574697.282150:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574697.282152:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574697.282155:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282158:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282160:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282162:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574697.282165:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574697.282166:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574697.282168:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574697.282170:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574697.282173:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574697.282175:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574697.282178:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282180:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282182:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282184:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574697.282216:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574697.282218:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282221:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574697.282223:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574697.282226:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574697.282229:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282231:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574697.282234:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574697.282236:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282238:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574697.282246:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574697.282249:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574697.282251:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574697.282253:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282255:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574697.282256:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574697.282258:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574697.282262:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574697.282266:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.282270:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810305536/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574697.282277:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574697.282280:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.282284:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810305536/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574697.282292:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574697.282295:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574697.282298:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574697.282300:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574697.282302:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574697.282304:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574697.282307:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15163 00000400:00000200:0.0:1788574697.282311:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574697.282314:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574697.282317:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574697.282320:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282322:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282324:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282327:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574697.282329:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574697.282334:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574697.282336:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574697.282338:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574697.282340:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574697.282343:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574697.282346:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282348:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282350:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282352:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574697.282355:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574697.282358:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282361:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574697.282362:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574697.282365:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574697.282367:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282369:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574697.282372:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574697.282374:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282376:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574697.282379:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574697.282382:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574697.282384:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574697.282386:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282387:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574697.282389:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574697.282391:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574697.282395:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574697.282398:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.282412:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810305664/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574697.282421:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574697.282424:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.282433:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810305664/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574697.282442:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574697.282445:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574697.282449:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574697.282451:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574697.282453:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574697.282456:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574697.282458:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15163 00000400:00000200:0.0:1788574697.282462:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574697.282465:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574697.282468:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574697.282471:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282473:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282475:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282477:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574697.282480:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574697.282481:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574697.282483:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574697.282485:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574697.282487:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574697.282489:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574697.282492:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282494:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574697.282497:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282499:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574697.282501:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574697.282504:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574697.282507:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574697.282512:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574697.282516:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574697.282518:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282520:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574697.282523:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574697.282525:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282527:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574697.282531:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574697.282533:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574697.282535:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574697.282537:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.282539:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574697.282540:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574697.282542:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574697.282546:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574697.282550:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.282555:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810305792/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574697.282563:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574697.282565:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.282569:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810305792/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574697.282579:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574697.282583:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15173) 00000020:00000001:0.0:1788574699.713840:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574699.713844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574699.713847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574700.737833:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574700.737837:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574700.737840:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574701.121694:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309840094, at 4309840128 00000100:00000010:0.0:1788574701.121711:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00080000:0.0:1788574702.401714:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574702.401748:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574702.401751:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574702.401756:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574702.401763:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574702.401773:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15168 00000400:00000200:0.0:1788574702.401785:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574702.401795:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574702.401800:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574702.401806:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.401811:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.401814:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.401820:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574702.401824:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574702.401827:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574702.401831:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574702.401836:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574702.401839:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574702.401844:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574702.401846:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.401850:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.401854:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.401858:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574702.401861:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574702.401865:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574702.401869:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574702.401874:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574702.401880:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574702.401886:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.401890:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574702.401894:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574702.401900:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574702.401910:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574702.401916:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574702.401918:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.401920:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574702.401923:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574702.401928:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574702.401936:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574702.401943:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.401953:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810305920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574702.401962:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574702.401966:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.401970:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810305920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.401989:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574702.401992:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574702.401995:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574702.401996:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574702.401998:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574702.402000:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574702.402002:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15168 00000400:00000200:0.0:1788574702.402006:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574702.402008:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574702.402011:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574702.402013:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402015:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402017:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402019:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574702.402021:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574702.402023:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574702.402026:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574702.402033:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574702.402036:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574702.402039:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574702.402043:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402048:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402052:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402057:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574702.402060:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574702.402065:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402069:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574702.402071:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574702.402079:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574702.402083:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402085:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574702.402089:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574702.402092:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402095:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574702.402098:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574702.402102:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574702.402109:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574702.402112:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402115:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574702.402117:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574702.402123:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574702.402128:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574702.402136:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.402145:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810306048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574702.402159:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574702.402162:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.402166:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810306048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.402185:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574702.402206:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574702.402211:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574702.402212:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574702.402215:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574702.402223:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574702.402226:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15168 00000400:00000200:0.0:1788574702.402230:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574702.402236:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574702.402241:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574702.402247:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402249:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402254:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402269:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574702.402273:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574702.402277:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574702.402280:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574702.402284:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574702.402286:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574702.402291:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574702.402296:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402299:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402300:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402302:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574702.402306:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574702.402311:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402315:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574702.402327:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574702.402348:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574702.402354:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402358:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574702.402362:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574702.402367:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402372:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574702.402377:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574702.402383:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574702.402386:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574702.402388:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402392:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574702.402395:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574702.402413:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574702.402419:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574702.402425:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.402432:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810306176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574702.402442:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574702.402446:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.402452:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810306176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.402472:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574702.402476:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574702.402484:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574702.402486:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574702.402490:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574702.402496:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574702.402498:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15168 00000400:00000200:0.0:1788574702.402505:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574702.402508:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574702.402512:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574702.402522:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402525:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402544:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402551:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574702.402553:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574702.402557:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574702.402558:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574702.402562:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574702.402567:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574702.402571:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574702.402577:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402580:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402582:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402584:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574702.402586:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574702.402588:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402603:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574702.402605:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574702.402608:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574702.402613:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402617:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574702.402621:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574702.402625:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402628:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574702.402633:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574702.402653:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574702.402659:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574702.402662:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402664:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574702.402667:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574702.402670:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574702.402676:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574702.402686:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.402691:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810306304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574702.402697:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574702.402699:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.402702:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810306304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.402712:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574702.402715:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574702.402719:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574702.402721:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574702.402734:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574702.402740:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574702.402745:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15168 00000400:00000200:0.0:1788574702.402750:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574702.402756:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574702.402759:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574702.402765:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402768:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402774:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402778:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574702.402782:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574702.402785:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574702.402788:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574702.402791:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574702.402795:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574702.402799:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574702.402806:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402810:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574702.402818:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402822:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574702.402825:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574702.402830:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574702.402836:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574702.402838:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574702.402840:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574702.402843:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402845:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574702.402846:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574702.402848:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402852:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574702.402859:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574702.402862:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574702.402866:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574702.402869:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.402874:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574702.402876:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574702.402877:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574702.402882:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574702.402887:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.402895:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810306432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574702.402908:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574702.402912:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.402918:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810306432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.402932:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574702.402937:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15179) 00000100:00080000:0.0:1788574704.386216:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.386227:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574704.386234:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.386237:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.386239:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.386243:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15174 00000400:00000200:0.0:1788574704.386249:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574704.386254:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.386257:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.386259:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386262:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386263:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386265:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574704.386268:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.386269:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574704.386270:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.386272:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574704.386274:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574704.386275:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574704.386277:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386294:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386296:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386297:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574704.386300:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574704.386302:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386304:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.386306:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.386308:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.386310:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386311:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574704.386313:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.386316:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574704.386318:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574704.386324:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.386326:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386327:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.386329:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.386331:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.386334:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574704.386337:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.386343:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810306688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.386349:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.386352:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.386356:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810306688/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.386375:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.386377:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.386401:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574704.386403:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.386404:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.386409:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.386411:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15174 00000400:00000200:0.0:1788574704.386415:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574704.386418:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.386420:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.386424:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386428:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386430:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386435:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574704.386438:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.386440:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574704.386442:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.386445:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574704.386453:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574704.386456:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574704.386479:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386483:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386488:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386493:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574704.386497:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574704.386501:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386506:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.386508:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.386512:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.386518:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386519:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.386525:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574704.386528:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386532:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.386537:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574704.386540:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574704.386542:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.386558:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386562:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.386566:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.386568:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.386574:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574704.386580:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.386606:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810306816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.386623:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.386628:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.386637:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810306816/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.386702:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.386709:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.386713:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574704.386716:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.386718:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.386721:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.386726:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15174 00000400:00000200:0.0:1788574704.386759:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574704.386766:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.386768:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.386771:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386775:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386777:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386780:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574704.386783:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.386785:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574704.386787:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.386790:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574704.386792:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574704.386796:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574704.386803:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386807:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386813:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386816:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574704.386819:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574704.386821:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386823:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.386833:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.386838:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.386841:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386843:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.386849:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574704.386854:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386856:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.386861:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574704.386863:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574704.386868:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.386870:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.386874:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.386876:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.386879:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.386885:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574704.386891:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.386900:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810306944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.386908:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.386912:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.386917:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810306944/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.386927:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.386930:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.386935:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574704.386937:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.386940:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.386943:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.386947:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15174 00000400:00000200:0.0:1788574704.386951:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574704.386957:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.386959:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.386968:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386970:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.386974:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.386978:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574704.386981:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.386982:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574704.386985:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.386987:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574704.386991:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574704.386995:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574704.387000:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387001:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.387003:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387006:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574704.387010:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574704.387013:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387031:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.387036:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.387040:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.387042:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387045:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.387049:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574704.387054:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387058:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.387062:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574704.387065:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574704.387067:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.387068:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387069:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.387070:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.387073:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.387078:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574704.387089:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.387096:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810307072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.387109:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.387113:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.387121:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810307072/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.387133:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.387136:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.387144:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574704.387147:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.387150:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.387154:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.387159:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15174 00000400:00000200:0.0:1788574704.387167:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574704.387172:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.387178:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.387184:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387188:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.387193:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387194:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574704.387196:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.387197:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574704.387198:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.387199:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574704.387200:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574704.387202:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574704.387212:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387214:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.387220:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387221:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574704.387223:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574704.387225:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574704.387231:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.387234:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.387239:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.387242:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387243:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.387245:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574704.387246:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387249:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.387256:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574704.387275:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574704.387278:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.387280:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.387284:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.387286:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.387289:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.387293:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574704.387296:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.387303:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810307200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.387315:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.387318:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.387325:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810307200/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.387340:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.387345:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15181) 00000020:00000001:0.0:1788574704.833800:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574704.833803:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574704.833806:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574705.857813:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574705.857817:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574705.857821:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574708.225709:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309847199, at 4309847232 00000100:00000010:0.0:1788574708.225723:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00080000:0.0:1788574709.569741:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574709.569752:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574709.569755:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574709.569759:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574709.569764:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574709.569771:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15176 00000400:00000200:0.0:1788574709.569781:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574709.569788:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574709.569792:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574709.569796:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.569799:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.569802:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.569804:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574709.569809:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574709.569811:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574709.569814:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574709.569817:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574709.569820:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574709.569823:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574709.569826:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.569828:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.569830:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.569833:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574709.569836:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574709.569840:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574709.569874:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.569878:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574709.569882:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.569885:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.569888:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574709.569891:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574709.569898:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574709.569902:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574709.569907:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574709.569909:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.569913:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574709.569915:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574709.569919:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574709.569928:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574709.569935:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.569963:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810307328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574709.569973:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574709.569978:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.569983:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810307328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.570009:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574709.570013:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574709.570016:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574709.570018:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574709.570020:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574709.570023:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574709.570026:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15176 00000400:00000200:0.0:1788574709.570030:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574709.570034:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574709.570037:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574709.570040:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570048:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570052:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570054:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574709.570057:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574709.570059:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574709.570061:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574709.570063:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574709.570066:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574709.570068:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574709.570089:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570095:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570101:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570104:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574709.570106:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574709.570112:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570119:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.570123:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574709.570130:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.570136:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570141:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574709.570146:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574709.570153:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570156:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574709.570162:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574709.570167:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574709.570172:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574709.570175:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570180:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574709.570183:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574709.570185:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574709.570193:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574709.570198:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.570226:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810307456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574709.570243:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574709.570248:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.570257:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810307456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.570275:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574709.570281:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574709.570291:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574709.570294:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574709.570297:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574709.570303:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574709.570310:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15176 00000400:00000200:0.0:1788574709.570318:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574709.570323:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574709.570326:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574709.570332:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570336:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570340:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570342:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574709.570347:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574709.570350:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574709.570352:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574709.570356:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574709.570361:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574709.570365:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574709.570372:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570376:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570380:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570399:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574709.570402:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574709.570409:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570413:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.570415:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574709.570418:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.570421:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570425:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574709.570427:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574709.570431:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570434:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574709.570438:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574709.570441:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574709.570444:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574709.570446:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570448:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574709.570450:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574709.570452:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574709.570457:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574709.570473:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.570480:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810307584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574709.570488:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574709.570492:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.570497:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810307584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.570507:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574709.570512:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574709.570514:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574709.570515:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574709.570518:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574709.570539:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574709.570545:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15176 00000400:00000200:0.0:1788574709.570553:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574709.570561:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574709.570567:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574709.570574:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570580:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570584:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570589:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574709.570596:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574709.570600:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574709.570604:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574709.570607:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574709.570611:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574709.570614:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574709.570621:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570625:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570628:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570632:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574709.570654:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574709.570657:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570660:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.570661:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574709.570664:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.570666:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570670:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574709.570674:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574709.570677:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570681:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574709.570687:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574709.570695:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574709.570710:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574709.570713:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570716:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574709.570719:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574709.570722:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574709.570726:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574709.570732:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.570740:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810307712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574709.570752:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574709.570756:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.570762:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810307712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.570769:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574709.570774:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574709.570778:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574709.570779:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574709.570780:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574709.570783:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574709.570787:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15176 00000400:00000200:0.0:1788574709.570792:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574709.570800:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574709.570803:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574709.570809:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570813:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570816:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570818:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574709.570822:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574709.570826:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574709.570836:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574709.570839:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574709.570843:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574709.570846:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574709.570851:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570855:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574709.570859:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570862:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574709.570865:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574709.570869:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574709.570873:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.570876:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574709.570880:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.570883:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570886:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574709.570889:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574709.570891:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570892:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574709.570895:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574709.570899:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574709.570903:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574709.570906:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.570908:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574709.570911:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574709.570914:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574709.570920:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574709.570926:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.570932:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810307840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574709.570939:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574709.570941:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.570947:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810307840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.570970:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.570975:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15186) 00000020:00000001:0.0:1788574709.953898:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574709.953901:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574709.953904:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574710.977829:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574710.977832:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574710.977835:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574713.409702:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309852382, at 4309852416 00000100:00000010:0.0:1788574713.409718:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788574715.073810:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574715.073814:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574715.073817:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574718.529754:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309857502, at 4309857536 00000100:00000010:0.0:1788574718.529774:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000800:00000200:0.0:1788574719.233984:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574720.193853:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574720.193860:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574720.193864:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574723.649713:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309862622, at 4309862656 00000100:00000010:0.0:1788574723.649728:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000100:00080000:0.0:1788574724.929774:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574724.929784:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574724.929786:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574724.929789:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574724.929792:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574724.929798:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15191 00000400:00000200:0.0:1788574724.929806:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574724.929811:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574724.929814:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574724.929816:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929835:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.929837:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929839:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574724.929842:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574724.929844:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574724.929846:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574724.929849:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574724.929851:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574724.929852:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574724.929854:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929856:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.929857:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929859:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574724.929861:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574724.929863:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929865:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574724.929867:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574724.929870:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574724.929872:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.929874:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574724.929877:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574724.929883:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574724.929887:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574724.929890:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574724.929892:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.929895:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574724.929898:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574724.929902:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574724.929911:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574724.929918:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.929928:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810309248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574724.929945:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574724.929948:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.929952:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810309248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574724.929977:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574724.929979:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574724.929982:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574724.929983:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574724.929985:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574724.929987:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574724.929988:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15191 00000400:00000200:0.0:1788574724.929992:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574724.929994:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574724.929996:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574724.929998:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.929999:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930001:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930002:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574724.930004:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574724.930005:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574724.930006:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574724.930007:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574724.930009:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574724.930010:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574724.930012:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930013:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930015:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930016:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574724.930022:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574724.930024:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930026:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574724.930028:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574724.930033:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574724.930035:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930038:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574724.930045:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574724.930051:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930055:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574724.930061:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574724.930066:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574724.930072:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574724.930075:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930079:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574724.930082:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574724.930086:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574724.930092:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574724.930097:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.930106:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810309376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574724.930117:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574724.930121:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.930128:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810309376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574724.930140:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574724.930144:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574724.930147:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574724.930150:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574724.930153:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574724.930159:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574724.930168:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15191 00000400:00000200:0.0:1788574724.930175:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574724.930179:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574724.930182:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574724.930187:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930189:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930191:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930193:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574724.930201:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574724.930203:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574724.930206:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574724.930209:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574724.930211:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574724.930219:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574724.930228:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930232:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930237:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930241:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574724.930244:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574724.930252:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930259:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574724.930265:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574724.930271:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574724.930275:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930279:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574724.930283:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574724.930286:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930290:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574724.930295:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574724.930299:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574724.930302:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574724.930309:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930312:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574724.930315:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574724.930317:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574724.930323:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574724.930330:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.930338:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810309504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574724.930351:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574724.930356:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.930361:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810309504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574724.930368:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574724.930372:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574724.930377:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574724.930378:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574724.930379:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574724.930381:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574724.930386:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15191 00000400:00000200:0.0:1788574724.930403:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574724.930407:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574724.930409:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574724.930416:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930418:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930422:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930424:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574724.930430:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574724.930431:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574724.930433:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574724.930436:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574724.930439:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574724.930449:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574724.930455:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930460:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930464:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930467:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574724.930471:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574724.930474:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930476:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574724.930480:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574724.930484:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574724.930488:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930490:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574724.930494:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574724.930497:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930498:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574724.930500:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574724.930505:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574724.930510:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574724.930512:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930513:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574724.930517:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574724.930519:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574724.930524:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574724.930529:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.930537:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810309632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574724.930544:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574724.930547:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.930553:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810309632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574724.930583:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574724.930589:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574724.930592:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574724.930596:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574724.930598:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574724.930601:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574724.930603:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15191 00000400:00000200:0.0:1788574724.930610:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574724.930615:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574724.930618:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574724.930622:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930625:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930627:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930630:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574724.930633:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574724.930634:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574724.930676:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574724.930678:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574724.930680:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574724.930683:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574724.930703:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930705:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574724.930707:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930709:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574724.930713:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574724.930716:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574724.930719:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574724.930722:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574724.930726:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574724.930728:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930732:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574724.930740:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574724.930747:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930749:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574724.930754:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574724.930756:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574724.930760:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574724.930763:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574724.930766:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574724.930769:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574724.930771:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574724.930776:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574724.930782:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.930789:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810309760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574724.930797:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574724.930801:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.930805:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810309760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574724.930817:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574724.930821:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15201) 00000020:00000001:0.0:1788574725.313842:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574725.313849:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574725.313855:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574730.113699:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309869087, at 4309869120 00000100:00000010:0.0:1788574730.113714:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574730.433807:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574730.433811:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574730.433815:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574731.586157:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574731.586176:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574731.586189:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574731.586205:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586240:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.586244:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.586249:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574731.586254:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574731.586259:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.586267:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810310656:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574731.586276:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574731.586283:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574731.586288:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.586295:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574731.586303:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.586308:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.586317:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574731.586322:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574731.586332:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574731.586340:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810310656, portal 25 00000100:00000001:0.0:1788574731.586344:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574731.586346:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788574731.586357:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586368:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574731.586373:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574731.586376:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574731.586379:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810310656, offset 0 00000400:00000200:0.0:1788574731.586389:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574731.586400:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.586410:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574731.586419:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574731.586425:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574731.586430:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574731.586441:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574731.586449:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542472:542472:256:4294967295] 192.168.202.148@tcp LPNI seq info [542437:542437:8:4294967295] 00000400:00000200:0.0:1788574731.586454:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.586466:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574731.586472:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.586479:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574731.586489:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574731.586495:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574731.586516:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.586522:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574731.586529:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574731.586538:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574731.586543:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574731.586547:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574731.586553:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574731.586559:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586574:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.586580:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586589:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.586590:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574731.586595:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.586601:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574731.586605:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4309874398 00000100:00000010:0.0:1788574731.586612:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788574731.586630:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574731.586661:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4309874399 00000100:00000010:0.0:1788574731.586665:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116263c0. 00000100:00000001:0.0:1788574731.586668:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574731.586670:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574731.586678:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.586682:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574731.586689:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574731/real 1788574731] req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574731.586702:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574731.586705:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574731.586710:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574731.586716:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574731.586719:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574731.586724:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574731.586729:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574731.586741:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586749:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574731.586751:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586756:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.586758:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574731.586766:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586777:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574731.586784:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586806:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574731.586811:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586813:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.586815:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586819:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574731.586821:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574731.586825:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574731.586831:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574731.586850:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574731.586857:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574731.586863:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810310656:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574731.586870:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586873:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.586877:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440810310656/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586893:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574731.586896:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574731.586901:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.586906:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574731.586910:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574731.586914:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574731.586919:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574731.586922:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d800. 02000000:00000001:0.0:1788574731.586927:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574731.586930:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574731.586933:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788574731.586937:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574731.586939:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586953:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.586961:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574731.586965:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.586970:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574731.586972:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574731.586975:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574731.586982:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.586995:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.586998:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587001:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574731.587005:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574731.587014:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.587019:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810310784:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574731.587024:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574731.587027:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574731.587030:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587032:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574731.587042:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.587043:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.587051:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574731.587056:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574731.587063:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574731.587069:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810310784, portal 10 00000100:00000001:0.0:1788574731.587074:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574731.587076:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000040:0.0:1788574731.587101:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587109:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574731.587114:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574731.587117:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574731.587121:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810310784, offset 0 00000400:00000200:0.0:1788574731.587127:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587133:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587151:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574731.587159:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574731.587165:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574731.587172:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574731.587179:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587188:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542473:542473:256:4294967295] 192.168.202.148@tcp LPNI seq info [542438:542438:8:4294967295] 00000400:00000200:0.0:1788574731.587202:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587212:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574731.587221:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587234:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574731.587245:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574731.587251:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574731.587280:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587285:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574731.587290:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574731.587298:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574731.587303:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574731.587307:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574731.587311:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574731.587316:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587327:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.587332:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587344:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587346:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574731.587351:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574731.587356:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788574731.587362:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574731.587365:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309870617 00000100:00000001:0.0:1788574731.587370:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309874398 00000100:00000001:0.0:1788574731.587374:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309870617 00000100:00000001:0.0:1788574731.587377:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309874399 00000100:00000001:0.0:1788574731.587382:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574731.587388:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587390:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574731.587396:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574731/real 1788574731] req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574731.587437:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574731.587440:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574731.587444:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574731.587448:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574731.587451:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574731.587454:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574731.587457:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574731.587466:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587488:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574731.587491:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587494:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.587496:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574731.587503:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587513:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574731.587519:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587537:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574731.587541:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587545:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.587547:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587552:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574731.587555:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574731.587557:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574731.587560:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574731.587561:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574731.587572:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574731.587578:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810310784:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574731.587581:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574731.587583:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587587:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.587588:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587590:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574731.587592:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574731.587593:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.587595:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810310912:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574731.587597:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574731.587599:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574731.587600:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587601:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574731.587614:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574731.587616:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.587618:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574731.587620:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574731.587622:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574731.587624:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810310912, portal 10 00000100:00000001:0.0:1788574731.587625:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574731.587626:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788574731.587630:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587650:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574731.587652:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574731.587653:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574731.587655:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810310912, offset 0 00000400:00000200:0.0:1788574731.587662:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587665:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587668:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574731.587670:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574731.587672:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574731.587674:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574731.587675:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587677:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542474:542474:256:4294967295] 192.168.202.148@tcp LPNI seq info [542439:542439:8:4294967295] 00000400:00000200:0.0:1788574731.587680:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587684:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574731.587687:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587690:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574731.587692:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574731.587694:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574731.587702:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587703:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574731.587705:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574731.587707:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574731.587709:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574731.587711:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574731.587712:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574731.587715:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587720:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.587722:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587726:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587727:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574731.587729:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574731.587732:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788574731.587734:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574731.587738:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309870622 00000100:00000001:0.0:1788574731.587740:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309874398 00000100:00000001:0.0:1788574731.587742:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309870622 00000100:00000001:0.0:1788574731.587743:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309874399 00000100:00000001:0.0:1788574731.587745:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574731.587746:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587747:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574731.587750:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574731/real 1788574731] req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574731.587755:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574731.587756:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574731.587757:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574731.587759:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574731.587760:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574731.587761:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574731.587763:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574731.587769:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587772:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574731.587774:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587775:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.587776:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574731.587778:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587783:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574731.587785:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587789:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574731.587793:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587794:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.587795:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587796:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574731.587798:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574731.587799:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574731.587801:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574731.587803:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574731.587805:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574731.587810:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810310912:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574731.587813:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587814:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.587816:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440810310784/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587820:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574731.587821:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574731.587823:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.587824:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574731.587825:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574731.587826:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574731.587827:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574731.587828:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788574731.587830:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574731.587831:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574731.587832:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788574731.587834:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574731.587834:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587835:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.587837:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810310912/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587850:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574731.587850:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574731.587852:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574731.587853:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574731.587854:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574731.587855:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574731.587856:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574731.587856:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788574731.587858:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574731.587858:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574731.587859:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788574731.587860:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574731.587861:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587862:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.587865:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574731.587866:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587867:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574731.587868:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574731.587869:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574731.587871:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587875:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.587876:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587878:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574731.587879:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574731.587880:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.587882:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810311040:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574731.587884:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574731.587885:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574731.587886:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.587887:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574731.587890:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574731.587891:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.587896:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574731.587898:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574731.587900:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574731.587902:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810311040, portal 4 00000100:00000001:0.0:1788574731.587904:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574731.587905:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788574731.587909:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587913:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574731.587915:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574731.587916:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574731.587917:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810311040, offset 0 00000400:00000200:0.0:1788574731.587920:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587922:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587925:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574731.587927:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574731.587928:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574731.587930:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574731.587932:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574731.587934:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542475:542475:256:4294967295] 192.168.202.148@tcp LPNI seq info [542440:542440:8:4294967295] 00000400:00000200:0.0:1788574731.587936:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.587940:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574731.587943:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587945:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574731.587947:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574731.587949:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574731.587956:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.587957:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574731.587959:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574731.587961:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574731.587963:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574731.587967:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574731.587981:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574731.587983:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587989:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.587991:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.587995:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.587996:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574731.587998:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574731.588000:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788574731.588002:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574731.588003:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309870754 00000100:00000001:0.0:1788574731.588004:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309874398 00000100:00000001:0.0:1788574731.588006:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309870754 00000100:00000001:0.0:1788574731.588007:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309874399 00000100:00000001:0.0:1788574731.588009:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574731.588010:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.588011:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574731.588013:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574731/real 1788574731] req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574731.588018:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574731.588019:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574731.588020:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574731.588022:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574731.588023:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574731.588024:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574731.588026:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574731.588031:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588049:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574731.588050:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588051:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.588052:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574731.588055:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588059:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574731.588062:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588066:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574731.588067:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.588069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588070:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574731.588072:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574731.588073:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574731.588075:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574731.588076:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574731.588079:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574731.588083:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810311040:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574731.588099:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574731.588102:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588106:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.588107:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.588112:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574731.588114:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574731.588115:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.588117:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810311168:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574731.588120:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574731.588121:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574731.588122:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574731.588123:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574731.588125:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.588136:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574731.588140:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574731.588143:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574731.588146:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574731.588149:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810311168, portal 4 00000100:00000001:0.0:1788574731.588151:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574731.588153:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788574731.588159:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588166:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574731.588169:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574731.588171:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574731.588174:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810311168, offset 0 00000400:00000200:0.0:1788574731.588179:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574731.588183:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.588187:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574731.588190:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574731.588192:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574731.588194:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574731.588195:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574731.588197:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542476:542476:256:4294967295] 192.168.202.148@tcp LPNI seq info [542441:542441:8:4294967295] 00000400:00000200:0.0:1788574731.588200:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574731.588204:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574731.588212:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.588214:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574731.588218:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574731.588220:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574731.588232:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574731.588234:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574731.588237:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574731.588241:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574731.588244:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574731.588246:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574731.588249:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574731.588253:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588261:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.588265:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574756 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588270:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.588271:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574731.588274:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574731.588276:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788574731.588278:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574731.588280:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309870763 00000100:00000001:0.0:1788574731.588282:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309874398 00000100:00000001:0.0:1788574731.588283:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309870763 00000100:00000001:0.0:1788574731.588284:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309874399 00000100:00000001:0.0:1788574731.588286:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574731.588288:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.588289:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574731.588291:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574731/real 1788574731] req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574731.588301:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574731.588302:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574731.588304:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574731.588305:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574731.588306:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574731.588308:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574731.588310:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574731.588316:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588319:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574731.588320:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588321:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.588322:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574731.588324:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588328:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574731.588331:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588335:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574731.588337:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588338:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574731.588339:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588340:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574731.588341:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574731.588342:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574731.588344:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574731.588345:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574731.588352:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574731.588357:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810311168:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574731.588360:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588361:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.588363:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810311040/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588366:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574731.588368:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574731.588369:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20e000. 02000000:00000001:0.0:1788574731.588371:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574731.588372:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574731.588373:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788574731.588374:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574731.588375:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788574731.588377:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574731.588378:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574731.588379:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788574731.588391:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574731.588392:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588394:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574731.588395:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440810311168/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574756 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574731.588399:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574731.588400:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574731.588401:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d000. 02000000:00000001:0.0:1788574731.588402:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574731.588402:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574731.588403:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788574731.588404:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574731.588405:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788574731.588406:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574731.588410:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574731.588411:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788574731.588412:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574731.588413:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588414:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.588417:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574731.588418:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574731.588419:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574731.588420:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574735.425784:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309874398, at 4309874432 00000100:00000010:0.0:1788574735.425809:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000020:00000001:0.0:1788574735.553821:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574735.553826:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574735.553831:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574736.705736:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574736.705745:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574736.705747:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574736.705751:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574736.705761:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574736.705767:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15203 00000400:00000200:0.0:1788574736.705776:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574736.705782:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574736.705786:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574736.705790:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705794:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.705797:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705800:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574736.705804:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574736.705806:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574736.705809:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574736.705812:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574736.705815:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574736.705818:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574736.705838:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705840:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.705843:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705845:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574736.705848:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574736.705852:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705855:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574736.705858:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574736.705862:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574736.705865:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.705867:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574736.705870:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574736.705877:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574736.705881:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574736.705886:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574736.705887:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.705889:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574736.705892:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574736.705896:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574736.705902:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574736.705908:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.705916:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810311296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574736.705923:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574736.705927:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.705931:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810311296/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574736.705951:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574736.705953:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574736.705956:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574736.705957:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574736.705963:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574736.705966:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574736.705968:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15203 00000400:00000200:0.0:1788574736.705979:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574736.705984:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574736.705988:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574736.705992:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.705996:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.705999:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706004:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574736.706009:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574736.706012:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574736.706015:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574736.706019:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574736.706022:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574736.706026:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574736.706031:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706038:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706043:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706046:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574736.706050:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574736.706056:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706059:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574736.706062:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574736.706069:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574736.706073:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706074:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574736.706081:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574736.706084:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706089:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574736.706100:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574736.706103:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574736.706107:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574736.706108:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706110:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574736.706111:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574736.706112:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574736.706117:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574736.706120:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.706128:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810311424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574736.706137:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574736.706139:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.706145:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810311424/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574736.706177:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574736.706180:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574736.706183:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574736.706184:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574736.706188:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574736.706194:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574736.706200:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15203 00000400:00000200:0.0:1788574736.706210:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574736.706216:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574736.706218:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574736.706226:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706232:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706238:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706242:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574736.706245:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574736.706254:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574736.706257:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574736.706260:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574736.706265:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574736.706268:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574736.706275:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706277:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706282:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706286:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574736.706290:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574736.706297:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706301:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574736.706305:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574736.706308:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574736.706312:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706314:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574736.706346:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574736.706352:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706357:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574736.706365:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574736.706369:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574736.706376:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574736.706377:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706380:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574736.706382:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574736.706386:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574736.706395:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574736.706402:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.706411:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810311552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574736.706424:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574736.706429:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.706445:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810311552/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574736.706471:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574736.706476:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574736.706481:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574736.706485:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574736.706489:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574736.706493:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574736.706499:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15203 00000400:00000200:0.0:1788574736.706506:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574736.706517:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574736.706520:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574736.706527:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706532:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706534:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706538:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574736.706540:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574736.706543:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574736.706545:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574736.706548:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574736.706550:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574736.706553:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574736.706559:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706563:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706567:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706572:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574736.706579:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574736.706582:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706583:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574736.706595:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574736.706599:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574736.706603:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706607:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574736.706611:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574736.706614:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706616:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574736.706676:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574736.706681:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574736.706688:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574736.706691:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706695:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574736.706697:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574736.706701:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574736.706709:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574736.706717:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.706729:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810311680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574736.706747:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574736.706752:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.706761:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810311680/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574736.706778:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574736.706785:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574736.706792:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574736.706796:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574736.706800:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574736.706806:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574736.706811:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15203 00000400:00000200:0.0:1788574736.706822:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574736.706831:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574736.706836:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574736.706849:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706854:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706859:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706864:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574736.706869:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574736.706874:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574736.706877:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574736.706882:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574736.706888:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574736.706893:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574736.706902:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706908:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574736.706912:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706920:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574736.706927:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574736.706932:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574736.706938:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574736.706943:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574736.706951:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574736.706958:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706963:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574736.706968:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574736.706973:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.706977:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574736.706984:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574736.706990:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574736.706997:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574736.707000:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.707005:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574736.707007:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574736.707013:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574736.707021:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574736.707038:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.707050:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810311808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574736.707064:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574736.707069:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.707079:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810311808/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574736.707100:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574736.707109:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15213) 00000100:00000040:0.0:1788574740.545743:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309879518, at 4309879552 00000100:00000010:0.0:1788574740.545762:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788574740.673881:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574740.673885:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574740.673888:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574741.825755:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574741.825768:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574741.825770:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574741.825774:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574741.825779:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574741.825787:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15208 00000400:00000200:0.0:1788574741.825796:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574741.825802:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574741.825806:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574741.825808:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.825812:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.825815:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.825817:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574741.825822:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574741.825824:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574741.825827:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574741.825843:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574741.825846:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574741.825848:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574741.825852:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.825854:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.825856:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.825858:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574741.825861:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574741.825865:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574741.825868:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574741.825871:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574741.825875:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574741.825878:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.825881:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574741.825884:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574741.825890:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574741.825894:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574741.825898:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574741.825900:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.825903:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574741.825906:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574741.825911:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574741.825919:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574741.825925:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.825935:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810311936/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574741.825945:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574741.825950:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.825955:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810311936/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574741.825982:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574741.825986:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574741.825995:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574741.825996:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574741.825998:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574741.826001:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574741.826004:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15208 00000400:00000200:0.0:1788574741.826009:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574741.826012:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574741.826014:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574741.826017:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826019:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826021:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826023:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574741.826026:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574741.826027:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574741.826029:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574741.826031:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574741.826033:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574741.826035:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574741.826038:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826040:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826042:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826044:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574741.826046:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574741.826049:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826051:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574741.826053:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574741.826059:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574741.826061:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826063:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574741.826067:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574741.826076:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826077:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574741.826081:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574741.826084:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574741.826086:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574741.826088:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826090:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574741.826091:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574741.826093:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574741.826097:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574741.826101:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.826106:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810312064/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574741.826115:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574741.826117:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.826122:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810312064/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574741.826145:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574741.826151:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574741.826155:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574741.826245:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574741.826250:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574741.826256:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574741.826262:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15208 00000400:00000200:0.0:1788574741.826270:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574741.826277:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574741.826283:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574741.826289:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826291:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826294:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826304:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574741.826309:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574741.826311:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574741.826313:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574741.826315:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574741.826317:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574741.826319:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574741.826323:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826325:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826327:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826329:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574741.826332:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574741.826338:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826341:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574741.826347:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574741.826351:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574741.826354:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826359:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574741.826362:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574741.826365:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826370:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574741.826377:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574741.826382:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574741.826386:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574741.826389:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826394:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574741.826397:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574741.826401:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574741.826407:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574741.826415:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.826424:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810312192/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574741.826458:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574741.826463:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.826471:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810312192/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574741.826487:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574741.826493:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574741.826497:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574741.826499:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574741.826501:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574741.826505:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574741.826511:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15208 00000400:00000200:0.0:1788574741.826544:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574741.826551:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574741.826556:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574741.826560:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826566:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826572:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826576:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574741.826581:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574741.826584:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574741.826588:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574741.826593:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574741.826599:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574741.826604:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574741.826612:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826617:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826621:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826626:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574741.826680:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574741.826689:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826695:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574741.826700:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574741.826705:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574741.826710:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826715:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574741.826718:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574741.826725:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826729:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574741.826737:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574741.826743:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574741.826749:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574741.826751:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826753:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574741.826755:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574741.826759:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574741.826766:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574741.826772:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.826782:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810312320/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574741.826795:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574741.826799:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.826806:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810312320/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574741.826824:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574741.826829:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574741.826833:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574741.826834:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574741.826836:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574741.826838:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574741.826851:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15208 00000400:00000200:0.0:1788574741.826859:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574741.826866:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574741.826871:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574741.826880:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826885:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826890:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826894:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574741.826896:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574741.826901:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574741.826905:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574741.826909:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574741.826914:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574741.826920:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574741.826928:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826933:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574741.826938:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826944:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574741.826951:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574741.826957:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574741.826963:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574741.826966:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574741.826970:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574741.826975:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826980:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574741.826984:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574741.826990:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.826994:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574741.827000:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574741.827005:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574741.827012:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574741.827021:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.827025:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574741.827029:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574741.827032:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574741.827040:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574741.827048:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.827058:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810312448/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574741.827074:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574741.827080:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.827087:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810312448/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574741.827100:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574741.827105:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15218) 00000100:00000040:0.0:1788574745.665738:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309884639, at 4309884672 00000100:00000010:0.0:1788574745.665756:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788574745.793858:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574745.793863:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574745.793867:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574746.945727:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574746.945736:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574746.945738:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574746.945741:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574746.945745:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574746.945751:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15213 00000400:00000200:0.0:1788574746.945759:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574746.945764:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574746.945766:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574746.945769:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945772:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.945773:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945789:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574746.945792:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574746.945794:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574746.945796:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574746.945799:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574746.945801:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574746.945802:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574746.945805:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945806:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.945808:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945809:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574746.945811:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574746.945813:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945815:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574746.945817:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574746.945820:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574746.945822:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.945823:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574746.945825:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574746.945830:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574746.945832:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574746.945835:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574746.945836:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.945839:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574746.945840:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574746.945844:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574746.945859:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574746.945867:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.945874:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810312576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574746.945882:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574746.945885:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.945893:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810312576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574746.945912:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574746.945915:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574746.945917:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574746.945919:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574746.945920:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574746.945922:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574746.945923:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15213 00000400:00000200:0.0:1788574746.945926:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574746.945928:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574746.945930:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574746.945931:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945933:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.945934:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945936:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574746.945938:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574746.945939:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574746.945940:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574746.945942:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574746.945944:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574746.945946:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574746.945949:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945950:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.945952:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945954:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574746.945958:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574746.945961:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574746.945965:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574746.945972:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574746.945977:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574746.945983:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.945986:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574746.945991:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574746.945995:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.945998:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574746.946003:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574746.946009:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574746.946014:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574746.946016:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946020:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574746.946024:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574746.946028:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574746.946036:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574746.946042:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.946056:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810312704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574746.946067:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574746.946071:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.946075:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810312704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574746.946089:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574746.946093:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574746.946103:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574746.946107:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574746.946109:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574746.946114:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574746.946119:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15213 00000400:00000200:0.0:1788574746.946126:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574746.946132:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574746.946134:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574746.946149:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946152:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946156:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946160:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574746.946164:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574746.946168:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574746.946169:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574746.946172:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574746.946177:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574746.946181:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574746.946188:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946194:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946200:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946206:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574746.946211:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574746.946216:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946219:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574746.946230:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574746.946235:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574746.946241:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946245:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574746.946250:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574746.946253:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946257:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574746.946260:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574746.946266:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574746.946268:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574746.946270:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946274:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574746.946277:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574746.946280:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574746.946284:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574746.946296:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.946306:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810312832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574746.946314:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574746.946317:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.946325:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810312832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574746.946338:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574746.946342:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574746.946348:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574746.946352:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574746.946354:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574746.946360:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574746.946365:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15213 00000400:00000200:0.0:1788574746.946371:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574746.946374:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574746.946376:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574746.946377:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946379:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946380:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946385:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574746.946387:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574746.946388:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574746.946390:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574746.946392:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574746.946396:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574746.946400:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574746.946408:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946413:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946422:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946425:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574746.946427:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574746.946431:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946434:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574746.946438:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574746.946441:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574746.946444:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946447:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574746.946451:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574746.946453:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946455:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574746.946461:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574746.946464:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574746.946466:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574746.946469:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946472:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574746.946473:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574746.946475:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574746.946478:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574746.946484:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.946489:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810312960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574746.946500:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574746.946502:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.946508:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810312960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574746.946518:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574746.946521:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574746.946525:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574746.946527:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574746.946533:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574746.946537:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574746.946542:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15213 00000400:00000200:0.0:1788574746.946548:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574746.946551:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574746.946555:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574746.946560:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946565:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946569:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946575:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574746.946576:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574746.946579:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574746.946582:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574746.946590:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574746.946596:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574746.946601:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574746.946607:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946611:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574746.946615:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946618:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574746.946623:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574746.946630:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574746.946658:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574746.946661:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574746.946665:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574746.946668:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946670:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574746.946675:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574746.946682:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946685:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574746.946694:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574746.946710:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574746.946714:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574746.946716:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.946717:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574746.946718:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574746.946720:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574746.946724:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574746.946729:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.946737:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810313088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574746.946749:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574746.946755:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.946766:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810313088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574746.946787:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574746.946795:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15223) 10000000:00000001:0.0:1788574748.097780:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574748.097786:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574748.097791:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574748.097793:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574748.097795:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574748.097796:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574748.097798:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574748.097800:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574748.097802:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574748.097805:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574748.097806:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574748.097816:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574748.097820:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574748.097825:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574748.097828:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574748.097832:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574748.097838:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574748.097845:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574748.097861:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574748.097863:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574748.097870:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692700. 00010000:00000001:0.0:1788574748.097873:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574748.097876:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788574748.097880:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574748.097882:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x1441586093168c80 to hash 00000020:00000001:0.0:1788574748.097884:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574748.097885:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574748.097887:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574748.097890:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574748.097893:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574748.097896:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574748.097904:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574748.097913:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574748.097918:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574748.097919:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574748.097924:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574748.097925:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788574748.097927:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.097928:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788574748.097931:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574748.097934:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.097936:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574748.097937:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.097945:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.097955:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574748.097965:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.097972:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574748.097977:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.097985:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574748.097988:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574748.097990:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.097993:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574748.097996:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574748.097998:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574748.098002:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.098004:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.098005:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.098009:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.098015:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810313216:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574748.098020:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.098022:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574748.098023:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574748.098025:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.098027:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.098032:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574748.098034:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574748.098039:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574748.098040:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574748.098042:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574748.098044:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574748.098046:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574748.098053:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788574748.098054:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574748.098056:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00010000:0.0:1788574748.098059:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098064:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574748.098067:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098071:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574748.098073:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098076:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574748.098078:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574748.098079:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098083:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574748.098084:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574748.098086:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574748.098088:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574748.098095:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574748.098097:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574748.098100:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098105:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574748.098107:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574748.098113:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574748.098115:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=5 00000020:00000001:0.0:1788574748.098117:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574748.098119:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788574748.098123:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574748.098126:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574748.098133:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574748.098135:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574748.098141:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574748.098143:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574748.098144:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574748.098146:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574748.098148:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574748.098150:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574748.098151:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574748.098153:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574748.098154:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574748.098156:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30400 with handle 0x1441586093168c80 from hash 00010000:00000001:0.0:1788574748.098157:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574748.098158:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574748.098159:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574748.098160:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574748.098161:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574748.098162:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574748.098164:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574748.098165:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574748.098165:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574748.098167:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574748.098169:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574748.098179:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574748.098180:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574748.098181:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574748.098182:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574748.098183:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574748.098184:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574748.098184:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574748.098186:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168c80 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574748.098191:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692700 count: 0 00010000:00000001:0.0:1788574748.098197:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574748.098198:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574748.098200:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.098204:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810313216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.098209:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574748.098211:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.098212:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574748.098214:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.098216:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574748.098219:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.098221:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.098223:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788574748.098226:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.098228:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574748.098230:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574748.098232:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574748.098236:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000001:0.0:1788574748.098462:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574748.098467:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574748.098469:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.098478:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.098507:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.098509:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.098522:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574748.098527:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574748.098533:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.098545:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810313344:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574748.098554:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574748.098561:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574748.098566:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.098572:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574748.098581:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788574748.098585:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.098617:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574748.098623:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574748.098633:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574748.098708:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810313344, portal 25 00000100:00000001:0.0:1788574748.098713:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.098715:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291926656 : -109714417624960 : ffff9c371f1c5880) 00000100:00000040:0.0:1788574748.098726:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.098736:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574748.098743:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574748.098747:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574748.098758:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810313344, offset 0 00000400:00000200:0.0:1788574748.098768:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574748.098782:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.098789:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574748.098796:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574748.098806:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574748.098812:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574748.098817:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574748.098835:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542492:542492:256:4294967295] 192.168.202.148@tcp LPNI seq info [542457:542457:8:4294967295] 00000400:00000200:0.0:1788574748.098850:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.098871:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574748.098882:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.098892:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574748.098904:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574748.098909:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788574748.098940:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.098944:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574748.098952:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574748.098962:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574748.098973:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574748.098977:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574748.098984:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574748.098989:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099003:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.099010:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099031:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099036:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574748.099044:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.099051:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574748.099056:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4309890911 00000100:00000001:0.0:1788574748.099067:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4309889758 00000100:00000001:0.0:1788574748.099073:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4309890911 00000100:00000001:0.0:1788574748.099081:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4309889758 00000100:00000001:0.0:1788574748.099089:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574748.099095:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099106:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574748.099113:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574748/real 1788574748] req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574748.099143:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574748.099147:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574748.099151:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574748.099154:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574748.099156:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574748.099158:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574748.099166:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574748.099191:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099216:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574748.099221:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099224:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.099229:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574748.099238:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099246:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.099257:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099274:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574748.099281:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099285:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.099287:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099293:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574748.099296:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574748.099301:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574748.099311:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574748.099335:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574748.099345:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.099367:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810313344:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574748.099382:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099387:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.099395:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5880 x1875440810313344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099407:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574748.099411:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574748.099415:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788574748.099419:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574748.099423:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.099426:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574748.099431:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.099435:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233dc00. 02000000:00000001:0.0:1788574748.099441:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.099445:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.099448:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5880. 00000100:00000001:0.0:1788574748.099453:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.099455:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099460:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099469:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574748.099472:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099475:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574748.099478:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574748.099479:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.099484:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099492:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.099495:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.099498:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574748.099501:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574748.099513:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.099524:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313472:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574748.099533:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574748.099537:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574748.099540:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.099542:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574748.099546:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788574748.099549:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.099554:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574748.099558:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574748.099563:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574748.099567:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810313472, portal 10 00000100:00000001:0.0:1788574748.099569:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.099571:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932032 : -109714417619584 : ffff9c371f1c6d80) 00000100:00000040:0.0:1788574748.099579:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099583:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574748.099585:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574748.099587:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574748.099589:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810313472, offset 0 00000400:00000200:0.0:1788574748.099606:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574748.099610:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.099613:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574748.099616:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574748.099617:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574748.099619:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574748.099621:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574748.099623:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542493:542493:256:4294967295] 192.168.202.148@tcp LPNI seq info [542458:542458:8:4294967295] 00000400:00000200:0.0:1788574748.099626:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.099631:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574748.099634:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.099666:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574748.099671:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574748.099674:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788574748.099687:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.099689:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574748.099693:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574748.099697:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574748.099700:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574748.099701:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574748.099703:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574748.099706:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099712:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.099714:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099719:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099720:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574748.099722:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574748.099725:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788574748.099728:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574748.099729:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309887130 00000100:00000001:0.0:1788574748.099731:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.099733:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309887130 00000100:00000001:0.0:1788574748.099734:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.099736:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574748.099737:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099738:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574748.099741:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574748/real 1788574748] req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574748.099755:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574748.099756:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574748.099757:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574748.099759:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574748.099760:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574748.099762:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574748.099764:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574748.099770:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099774:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574748.099775:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099776:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.099777:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574748.099779:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099783:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.099785:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099790:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574748.099792:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099793:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.099794:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.099795:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574748.099796:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574748.099798:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574748.099800:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574748.099801:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574748.099807:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.099812:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313472:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574748.099815:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.099816:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099820:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.099821:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.099823:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574748.099824:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574748.099826:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.099827:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574748.099830:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574748.099831:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574748.099832:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.099833:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574748.099836:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788574748.099837:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.099839:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574748.099841:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574748.099843:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574748.099846:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810313600, portal 10 00000100:00000001:0.0:1788574748.099847:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.099848:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291930240 : -109714417621376 : ffff9c371f1c6680) 00000100:00000040:0.0:1788574748.099852:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099856:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574748.099858:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574748.099859:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574748.099861:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810313600, offset 0 00000400:00000200:0.0:1788574748.099867:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574748.099870:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.099872:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574748.099874:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574748.099876:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574748.099878:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574748.099880:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574748.099882:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542494:542494:256:4294967295] 192.168.202.148@tcp LPNI seq info [542459:542459:8:4294967295] 00000400:00000200:0.0:1788574748.099885:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.099889:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574748.099892:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.099894:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574748.099897:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574748.099899:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788574748.099906:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.099907:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574748.099909:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574748.099911:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574748.099914:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574748.099915:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574748.099917:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574748.099921:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099929:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.099933:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.099939:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099940:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574748.099944:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574748.099948:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788574748.099951:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574748.099959:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309887134 00000100:00000001:0.0:1788574748.099962:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.099964:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309887134 00000100:00000001:0.0:1788574748.099967:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.099969:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574748.099971:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.099973:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574748.099978:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574748/real 1788574748] req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574748.099984:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574748.099985:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574748.099986:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574748.099988:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574748.099989:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574748.099991:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574748.099993:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574748.099999:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100002:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574748.100003:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100004:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100005:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574748.100007:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100012:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.100014:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100019:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574748.100024:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100025:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100026:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100027:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574748.100028:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574748.100030:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574748.100033:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574748.100034:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574748.100037:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.100042:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313600:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574748.100044:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.100046:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100050:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.100051:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.100053:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574748.100054:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574748.100055:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.100057:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313728:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574748.100059:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574748.100060:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574748.100061:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.100062:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100064:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788574748.100066:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.100069:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574748.100071:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574748.100073:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574748.100075:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810313728, portal 4 00000100:00000001:0.0:1788574748.100080:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.100081:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291927552 : -109714417624064 : ffff9c371f1c5c00) 00000100:00000040:0.0:1788574748.100086:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100090:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574748.100092:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574748.100093:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574748.100095:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810313728, offset 0 00000400:00000200:0.0:1788574748.100097:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574748.100100:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.100103:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574748.100105:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574748.100107:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574748.100108:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574748.100110:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574748.100112:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542495:542495:256:4294967295] 192.168.202.148@tcp LPNI seq info [542460:542460:8:4294967295] 00000400:00000200:0.0:1788574748.100115:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.100119:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574748.100122:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.100124:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574748.100127:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574748.100129:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788574748.100136:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.100138:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574748.100140:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574748.100144:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574748.100147:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574748.100148:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574748.100151:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574748.100154:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100174:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100179:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100185:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100186:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574748.100190:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574748.100192:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788574748.100195:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574748.100196:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309887266 00000100:00000001:0.0:1788574748.100198:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.100199:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309887266 00000100:00000001:0.0:1788574748.100201:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.100202:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574748.100204:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100205:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574748.100207:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574748/real 1788574748] req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574748.100212:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574748.100213:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574748.100215:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574748.100216:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574748.100217:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574748.100219:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574748.100221:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574748.100226:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100230:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574748.100235:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100236:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100237:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574748.100240:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100244:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.100247:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100251:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574748.100252:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100253:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100254:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100255:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574748.100257:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574748.100258:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574748.100260:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574748.100262:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574748.100264:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.100269:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313728:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574748.100271:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100272:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100274:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6d80 x1875440810313472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100279:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574748.100280:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100281:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788574748.100282:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574748.100283:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.100287:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574748.100288:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.100290:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788574748.100292:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.100293:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.100294:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6d80. 00000100:00000001:0.0:1788574748.100296:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.100297:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100298:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100301:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6680 x1875440810313600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100305:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574748.100306:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100307:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788574748.100308:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574748.100308:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.100309:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574748.100310:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.100311:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788574748.100312:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.100313:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.100314:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6680. 00000100:00000001:0.0:1788574748.100315:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.100316:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100317:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100318:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5c00 x1875440810313728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100322:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574748.100323:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100324:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788574748.100325:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574748.100326:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.100327:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788574748.100328:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.100328:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788574748.100330:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.100333:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.100334:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5c00. 00000100:00000001:0.0:1788574748.100336:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.100336:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100338:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100341:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574748.100342:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100343:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574748.100344:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574748.100345:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574748.100347:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100352:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.100353:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.100355:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574748.100356:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574748.100357:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.100359:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574748.100361:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574748.100362:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574748.100363:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574748.100364:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100366:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788574748.100367:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574748.100369:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574748.100371:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574748.100373:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574748.100375:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810313856, portal 4 00000100:00000001:0.0:1788574748.100376:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574748.100377:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788574748.100381:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100385:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574748.100387:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574748.100391:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574748.100393:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810313856, offset 0 00000400:00000200:0.0:1788574748.100396:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574748.100398:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.100401:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574748.100403:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574748.100405:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574748.100406:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574748.100408:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574748.100410:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542496:542496:256:4294967295] 192.168.202.148@tcp LPNI seq info [542461:542461:8:4294967295] 00000400:00000200:0.0:1788574748.100413:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574748.100417:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574748.100420:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.100422:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574748.100424:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574748.100426:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788574748.100436:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574748.100438:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574748.100440:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574748.100442:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574748.100444:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574748.100445:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574748.100447:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574748.100449:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100454:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100456:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574773 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100460:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100461:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574748.100467:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574748.100469:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788574748.100472:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574748.100473:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309887275 00000100:00000001:0.0:1788574748.100475:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.100476:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309887275 00000100:00000001:0.0:1788574748.100478:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309890911 00000100:00000001:0.0:1788574748.100479:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574748.100480:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100482:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574748.100484:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574748/real 1788574748] req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574748.100489:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574748.100490:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574748.100491:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574748.100493:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574748.100494:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574748.100495:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574748.100497:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574748.100503:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100506:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574748.100507:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100508:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100509:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574748.100511:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100516:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574748.100520:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100531:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574748.100534:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100536:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574748.100537:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100539:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574748.100541:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574748.100544:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574748.100547:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574748.100550:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574748.100555:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574748.100564:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810313856:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574748.100567:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100569:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574748.100573:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810313856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574773 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574748.100577:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574748.100579:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574748.100580:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788574748.100582:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574748.100583:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574748.100584:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788574748.100585:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574748.100586:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788574748.100588:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574748.100589:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574748.100590:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788574748.100610:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574748.100611:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100616:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.100619:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574748.100620:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.100621:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574748.100622:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574750.913882:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574750.913889:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574750.913893:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574751.937783:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309890911, at 4309890944 00000100:00000010:0.0:1788574751.937799:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00080000:0.0:1788574753.601791:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574753.601808:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574753.601812:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574753.601817:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574753.601823:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574753.601833:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15219 00000400:00000200:0.0:1788574753.601846:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574753.601854:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574753.601859:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574753.601862:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.601865:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.601867:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.601869:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574753.601873:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574753.601875:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574753.601878:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574753.601882:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574753.601885:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574753.601888:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574753.601891:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.601893:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.601896:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.601925:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574753.601929:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574753.601932:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574753.601934:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574753.601937:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574753.601940:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574753.601943:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.601945:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574753.601947:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574753.601954:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574753.601957:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574753.601961:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574753.601963:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.601966:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574753.601970:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574753.601974:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574753.601982:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574753.601989:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.601999:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810313984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574753.602008:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574753.602013:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.602017:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810313984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574753.602043:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574753.602046:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574753.602049:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574753.602050:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574753.602051:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574753.602053:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574753.602055:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15219 00000400:00000200:0.0:1788574753.602064:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574753.602068:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574753.602070:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574753.602075:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602077:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602081:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602085:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574753.602088:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574753.602090:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574753.602092:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574753.602094:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574753.602098:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574753.602102:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574753.602109:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602112:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602117:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602122:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574753.602125:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574753.602132:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602139:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574753.602145:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574753.602155:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574753.602159:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602162:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574753.602167:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574753.602173:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602178:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574753.602185:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574753.602187:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574753.602193:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574753.602201:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602206:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574753.602208:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574753.602210:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574753.602215:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574753.602221:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.602226:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810314112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574753.602237:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574753.602241:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.602247:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810314112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574753.602273:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574753.602279:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574753.602283:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574753.602285:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574753.602288:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574753.602292:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574753.602296:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15219 00000400:00000200:0.0:1788574753.602302:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574753.602306:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574753.602308:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574753.602310:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602315:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602321:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602323:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574753.602326:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574753.602327:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574753.602331:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574753.602334:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574753.602337:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574753.602346:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574753.602352:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602357:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602362:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602366:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574753.602371:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574753.602377:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602382:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574753.602383:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574753.602386:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574753.602390:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602413:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574753.602415:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574753.602417:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602418:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574753.602421:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574753.602424:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574753.602430:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574753.602432:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602434:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574753.602435:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574753.602440:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574753.602448:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574753.602456:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.602466:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810314240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574753.602489:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574753.602494:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.602501:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810314240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574753.602508:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574753.602520:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574753.602526:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574753.602529:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574753.602533:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574753.602536:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574753.602539:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15219 00000400:00000200:0.0:1788574753.602556:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574753.602562:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574753.602567:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574753.602573:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602579:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602581:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602586:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574753.602590:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574753.602592:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574753.602595:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574753.602599:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574753.602604:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574753.602606:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574753.602612:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602613:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602615:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602617:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574753.602622:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574753.602626:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602630:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574753.602633:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574753.602690:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574753.602695:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602698:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574753.602707:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574753.602711:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602713:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574753.602717:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574753.602722:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574753.602727:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574753.602730:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602734:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574753.602737:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574753.602741:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574753.602746:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574753.602752:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.602758:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810314368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574753.602777:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574753.602781:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.602789:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810314368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574753.602808:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574753.602815:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574753.602824:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574753.602828:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574753.602834:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574753.602842:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574753.602849:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15219 00000400:00000200:0.0:1788574753.602857:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574753.602864:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574753.602868:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574753.602872:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602876:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602882:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602894:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574753.602900:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574753.602904:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574753.602908:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574753.602913:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574753.602919:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574753.602926:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574753.602933:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602937:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574753.602942:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602944:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574753.602951:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574753.602957:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574753.602961:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574753.602965:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574753.602968:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574753.602970:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602971:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574753.602973:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574753.602975:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602979:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574753.602987:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574753.602989:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574753.602992:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574753.602994:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.602998:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574753.603001:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574753.603003:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574753.603009:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574753.603014:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.603021:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810314496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574753.603049:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574753.603053:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.603059:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810314496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574753.603069:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574753.603074:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15230) 00000020:00000001:0.0:1788574756.033884:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574756.033888:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574756.033892:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574757.121964:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574757.441717:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309896414, at 4309896448 00000100:00000010:0.0:1788574757.441744:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00000001:0.0:1788574758.722055:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574758.722060:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574758.722062:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574758.722073:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722088:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.722091:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.722098:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574758.722101:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574758.722106:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.722113:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810314624:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574758.722119:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574758.722123:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574758.722135:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.722139:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574758.722145:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.722147:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.722156:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574758.722160:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574758.722171:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574758.722179:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810314624, portal 25 00000100:00000001:0.0:1788574758.722183:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574758.722186:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286341504 : -109714423210112 : ffff9c371ec71f80) 00000100:00000040:0.0:1788574758.722192:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722205:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574758.722211:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574758.722215:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574758.722220:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810314624, offset 0 00000400:00000200:0.0:1788574758.722229:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574758.722236:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.722258:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574758.722265:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574758.722270:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574758.722276:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574758.722282:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574758.722290:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542502:542502:256:4294967295] 192.168.202.148@tcp LPNI seq info [542467:542467:8:4294967295] 00000400:00000200:0.0:1788574758.722300:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.722315:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574758.722331:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.722337:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574758.722348:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574758.722354:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788574758.722375:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.722377:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574758.722381:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574758.722388:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574758.722395:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574758.722408:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574758.722416:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574758.722433:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722458:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.722465:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722481:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.722486:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574758.722495:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.722506:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574758.722513:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4309901534 00000100:00000010:0.0:1788574758.722523:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116265a0. 00000100:00000001:0.0:1788574758.722533:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574758.722544:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4309901534 00000100:00000010:0.0:1788574758.722551:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116266c0. 00000100:00000001:0.0:1788574758.722557:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574758.722564:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574758.722568:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.722574:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574758.722582:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574758/real 1788574758] req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574758.722597:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574758.722600:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574758.722604:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574758.722609:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574758.722612:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574758.722617:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574758.722623:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574758.722657:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722663:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574758.722669:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722673:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.722676:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574758.722683:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722689:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574758.722705:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722720:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574758.722725:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722730:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.722735:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722738:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574758.722741:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574758.722745:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574758.722752:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574758.722764:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574758.722771:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574758.722792:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810314624:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788574758.722806:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722811:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.722818:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810314624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722828:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574758.722831:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574758.722835:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.722838:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574758.722842:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574758.722852:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574758.722856:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574758.722861:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2ec00. 02000000:00000001:0.0:1788574758.722865:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574758.722867:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574758.722872:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788574758.722878:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574758.722880:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722884:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.722891:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574758.722895:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.722901:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574758.722902:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574758.722904:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574758.722910:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.722919:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.722921:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.722926:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574758.722928:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574758.722933:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.722940:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810314752:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574758.722946:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574758.722950:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574758.722953:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.722957:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574758.722960:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.722964:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.722971:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574758.722977:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574758.722980:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574758.722985:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810314752, portal 10 00000100:00000001:0.0:1788574758.722988:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574758.722992:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286333440 : -109714423218176 : ffff9c371ec70000) 00000100:00000040:0.0:1788574758.723002:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723021:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574758.723027:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574758.723030:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574758.723033:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810314752, offset 0 00000400:00000200:0.0:1788574758.723041:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723049:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723055:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574758.723059:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574758.723065:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574758.723069:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574758.723073:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723076:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542503:542503:256:4294967295] 192.168.202.148@tcp LPNI seq info [542468:542468:8:4294967295] 00000400:00000200:0.0:1788574758.723079:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723088:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574758.723098:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723105:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574758.723113:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574758.723119:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788574758.723139:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723143:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574758.723146:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574758.723152:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574758.723157:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574758.723162:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574758.723168:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574758.723173:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723187:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723190:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723214:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723217:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574758.723221:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574758.723227:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788574758.723232:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574758.723235:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309897753 00000100:00000001:0.0:1788574758.723254:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723258:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309897753 00000100:00000001:0.0:1788574758.723261:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723265:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574758.723269:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723274:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574758.723281:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574758/real 1788574758] req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574758.723294:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574758.723297:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574758.723300:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574758.723304:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574758.723307:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574758.723313:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574758.723317:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574758.723329:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723337:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574758.723338:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723339:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723340:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574758.723343:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723354:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574758.723357:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723362:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574758.723364:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723366:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723367:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723368:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574758.723369:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574758.723371:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574758.723373:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574758.723374:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574758.723377:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574758.723382:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810314752:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574758.723385:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574758.723387:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723391:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.723392:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723393:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574758.723394:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574758.723396:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.723397:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810314880:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574758.723399:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574758.723401:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574758.723404:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723406:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574758.723409:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788574758.723411:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.723415:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574758.723418:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574758.723422:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574758.723425:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810314880, portal 10 00000100:00000001:0.0:1788574758.723427:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574758.723428:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286337024 : -109714423214592 : ffff9c371ec70e00) 00000100:00000040:0.0:1788574758.723435:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723441:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574758.723444:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574758.723446:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574758.723449:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810314880, offset 0 00000400:00000200:0.0:1788574758.723453:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723458:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723462:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574758.723465:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574758.723467:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574758.723469:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574758.723471:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723473:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542504:542504:256:4294967295] 192.168.202.148@tcp LPNI seq info [542469:542469:8:4294967295] 00000400:00000200:0.0:1788574758.723476:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723481:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574758.723484:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723487:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574758.723490:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574758.723492:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788574758.723499:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723504:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574758.723507:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574758.723510:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574758.723512:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574758.723513:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574758.723515:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574758.723518:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723524:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723526:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723530:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723531:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574758.723534:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574758.723536:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788574758.723538:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574758.723539:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309897757 00000100:00000001:0.0:1788574758.723541:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723543:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309897757 00000100:00000001:0.0:1788574758.723544:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723546:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574758.723547:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723548:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574758.723550:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574758/real 1788574758] req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574758.723555:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574758.723556:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574758.723558:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574758.723559:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574758.723560:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574758.723565:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574758.723567:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574758.723573:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723576:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574758.723577:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723579:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723579:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574758.723582:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723586:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574758.723588:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723592:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574758.723594:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723595:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723596:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723597:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574758.723599:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574758.723600:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574758.723602:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574758.723603:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574758.723605:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574758.723610:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810314880:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788574758.723612:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723617:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723620:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70000 x1875440810314752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723624:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574758.723625:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574758.723627:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.723628:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574758.723629:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574758.723630:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574758.723632:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574758.723633:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2d400. 02000000:00000001:0.0:1788574758.723650:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574758.723652:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574758.723653:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70000. 00000100:00000001:0.0:1788574758.723655:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574758.723655:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723657:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723659:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70e00 x1875440810314880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723663:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574758.723664:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574758.723665:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788574758.723667:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574758.723667:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574758.723668:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788574758.723669:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574758.723670:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2dc00. 02000000:00000001:0.0:1788574758.723671:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574758.723672:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574758.723673:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70e00. 00000100:00000001:0.0:1788574758.723674:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574758.723675:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723676:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723679:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574758.723680:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723681:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574758.723685:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574758.723686:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574758.723689:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723694:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.723695:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723697:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574758.723698:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574758.723699:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.723701:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810315008:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574758.723703:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574758.723705:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574758.723706:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723707:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574758.723709:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788574758.723710:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.723713:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574758.723714:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574758.723717:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574758.723719:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810315008, portal 4 00000100:00000001:0.0:1788574758.723721:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574758.723722:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788574758.723726:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723730:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574758.723731:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574758.723733:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574758.723734:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810315008, offset 0 00000400:00000200:0.0:1788574758.723737:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723739:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723742:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574758.723744:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574758.723746:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574758.723750:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574758.723752:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723754:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542505:542505:256:4294967295] 192.168.202.148@tcp LPNI seq info [542470:542470:8:4294967295] 00000400:00000200:0.0:1788574758.723757:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723761:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574758.723764:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723766:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574758.723769:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574758.723771:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788574758.723778:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723779:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574758.723781:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574758.723783:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574758.723785:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574758.723786:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574758.723788:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574758.723791:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723796:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723798:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723802:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723803:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574758.723805:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574758.723807:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788574758.723809:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574758.723811:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309897890 00000100:00000001:0.0:1788574758.723812:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723814:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309897890 00000100:00000001:0.0:1788574758.723819:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.723821:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574758.723822:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.723824:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574758.723826:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574758/real 1788574758] req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574758.723832:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574758.723833:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574758.723834:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574758.723835:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574758.723836:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574758.723838:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574758.723840:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574758.723845:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723849:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574758.723850:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723851:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723852:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574758.723854:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723858:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574758.723860:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723864:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574758.723866:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723867:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.723868:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.723872:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574758.723874:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574758.723875:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574758.723878:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574758.723879:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574758.723882:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574758.723887:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810315008:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574758.723889:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574758.723891:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723895:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.723896:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723897:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574758.723899:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574758.723900:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.723901:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810315136:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574758.723903:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574758.723904:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574758.723905:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574758.723906:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574758.723908:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.723909:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574758.723911:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788574758.723913:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788574758.723915:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788574758.723917:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810315136, portal 4 00000100:00000001:0.0:1788574758.723918:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574758.723919:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286334336 : -109714423217280 : ffff9c371ec70380) 00000100:00000040:0.0:1788574758.723923:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723930:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574758.723932:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788574758.723933:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788574758.723935:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810315136, offset 0 00000400:00000200:0.0:1788574758.723937:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723940:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723942:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788574758.723944:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574758.723946:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788574758.723947:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574758.723949:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788574758.723951:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [542506:542506:256:4294967295] 192.168.202.148@tcp LPNI seq info [542471:542471:8:4294967295] 00000400:00000200:0.0:1788574758.723953:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574758.723957:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788574758.723960:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723962:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788574758.723964:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788574758.723966:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 41 secs later) 00000800:00000100:0.0:1788574758.723973:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788574758.723974:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788574758.723976:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574758.723979:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574758.723980:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788574758.723981:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788574758.723983:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574758.723985:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.723990:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.723992:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574783 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724004:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.724005:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574758.724007:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574758.724009:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788574758.724011:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788574758.724013:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4309897899 00000100:00000001:0.0:1788574758.724014:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.724016:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4309897899 00000100:00000001:0.0:1788574758.724017:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4309901534 00000100:00000001:0.0:1788574758.724018:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574758.724020:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.724021:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574758.724023:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574758/real 1788574758] req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574758.724028:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574758.724028:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574758.724029:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788574758.724031:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788574758.724032:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788574758.724033:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574758.724035:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574758.724040:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724044:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574758.724045:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724046:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.724047:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574758.724049:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724057:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574758.724060:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724064:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574758.724066:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724067:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574758.724068:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724069:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574758.724070:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574758.724071:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574758.724073:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788574758.724074:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574758.724077:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574758.724082:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810315136:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788574758.724084:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724085:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.724088:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810315008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724091:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574758.724092:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574758.724094:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788574758.724095:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574758.724096:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574758.724097:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788574758.724098:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574758.724099:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2f800. 02000000:00000001:0.0:1788574758.724100:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574758.724104:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574758.724105:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788574758.724107:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574758.724108:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724109:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574758.724111:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec70380 x1875440810315136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574758.724115:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574758.724116:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574758.724117:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788574758.724118:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574758.724119:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574758.724120:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788574758.724121:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574758.724122:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c400. 02000000:00000001:0.0:1788574758.724124:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574758.724124:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574758.724125:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec70380. 00000100:00000001:0.0:1788574758.724127:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574758.724128:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724129:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.724131:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574758.724132:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.724133:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574758.724135:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574761.153852:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574761.153855:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574761.153858:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574762.561756:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309901534, at 4309901568 00000100:00000010:0.0:1788574762.561776:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37116265a0. 00000020:00000001:0.0:1788574766.273842:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574766.273847:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574766.273851:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574767.681750:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309906654, at 4309906688 00000100:00000010:0.0:1788574767.681781:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 10000000:00000001:0.0:1788574770.433762:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574770.433792:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574770.433797:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574770.433800:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574770.433801:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574770.433803:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574770.433806:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574770.433808:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574770.433810:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574770.433813:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574770.433815:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574770.433823:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574770.433828:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574770.433833:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574770.433837:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574770.433843:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574770.433851:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574770.433859:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574770.433862:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574770.433865:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574770.433873:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574770.433878:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574770.433895:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31400. 00000020:00000001:0.0:1788574770.433901:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574770.433904:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31400 with handle 0x1441586093168c87 to hash 00000020:00000001:0.0:1788574770.433906:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574770.433907:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574770.433911:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574770.433914:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574770.433918:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574770.433923:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574770.433935:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574770.433962:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574770.433970:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574770.433973:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574770.433982:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574770.433984:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788574770.433987:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574770.433988:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788574770.433993:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574770.433999:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574770.434000:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574770.434002:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574770.434014:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574770.434022:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574770.434025:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574770.434030:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574770.434033:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574770.434037:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574770.434039:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574770.434041:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574770.434050:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574770.434052:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574770.434057:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574770.434060:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.434062:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574770.434064:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574770.434072:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574770.434079:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810316544:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574770.434085:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.434087:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574770.434089:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574770.434091:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574770.434094:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574770.434099:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574770.434101:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574770.434107:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574770.434108:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574770.434110:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574770.434113:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574770.434115:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574770.434117:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=4 00000020:00000001:0.0:1788574770.434119:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574770.434122:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00010000:0.0:1788574770.434125:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434130:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574770.434136:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434143:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574770.434146:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434175:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574770.434178:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574770.434181:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434189:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574770.434191:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574770.434193:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574770.434196:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574770.434201:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574770.434204:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574770.434206:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434210:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574770.434211:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574770.434212:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574770.434213:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31400 refcount=5 00000020:00000001:0.0:1788574770.434214:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574770.434216:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686080 : -109714172865536 : ffff9c372db31400) 00010000:00000001:0.0:1788574770.434218:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574770.434220:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574770.434225:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574770.434226:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574770.434231:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574770.434237:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574770.434238:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574770.434241:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574770.434243:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574770.434246:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574770.434248:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574770.434249:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574770.434250:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574770.434253:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31400 with handle 0x1441586093168c87 from hash 00010000:00000001:0.0:1788574770.434255:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574770.434263:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574770.434264:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574770.434264:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574770.434266:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574770.434267:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574770.434269:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574770.434270:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574770.434271:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574770.434273:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574770.434275:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574770.434279:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574770.434280:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574770.434281:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574770.434283:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574770.434284:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574770.434285:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574770.434286:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574770.434289:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31400/0x1441586093168c87 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574770.434297:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574770.434308:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574770.434311:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574770.434314:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574770.434328:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810316544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574770.434336:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574770.434338:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574770.434340:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574770.434342:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574770.434345:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574770.434349:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574770.434351:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574770.434352:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788574770.434355:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574770.434356:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574770.434358:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574770.434360:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574770.434363:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574771.393880:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574771.393884:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574771.393887:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574774.273800:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309913247, at 4309913280 00000100:00000010:0.0:1788574774.273820:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00080000:0.0:1788574775.617789:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574775.617799:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574775.617802:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574775.617805:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574775.617809:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574775.617816:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15242 00000400:00000200:0.0:1788574775.617825:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574775.617831:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574775.617835:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574775.617839:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.617843:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.617845:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.617847:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574775.617868:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574775.617871:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574775.617874:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574775.617877:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574775.617879:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574775.617881:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574775.617885:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.617887:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.617889:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.617891:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574775.617894:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574775.617897:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574775.617899:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574775.617903:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574775.617906:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574775.617909:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.617912:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574775.617914:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574775.617920:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574775.617924:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574775.617927:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574775.617929:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.617931:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574775.617934:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574775.617938:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574775.617945:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574775.617951:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.617959:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810317312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574775.617968:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574775.617973:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.617978:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810317312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574775.618008:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574775.618016:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574775.618019:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574775.618021:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574775.618022:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574775.618025:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574775.618027:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15242 00000400:00000200:0.0:1788574775.618032:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574775.618036:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574775.618039:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574775.618043:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618047:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618051:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618056:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574775.618061:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574775.618064:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574775.618068:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574775.618072:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574775.618077:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574775.618082:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574775.618090:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618094:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618099:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618103:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574775.618109:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574775.618114:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618119:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574775.618128:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574775.618135:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574775.618140:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618142:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574775.618146:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574775.618151:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618154:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574775.618160:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574775.618165:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574775.618169:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574775.618172:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618176:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574775.618179:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574775.618183:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574775.618191:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574775.618196:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.618206:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810317440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574775.618219:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574775.618224:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.618231:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810317440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574775.618245:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574775.618249:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574775.618255:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574775.618259:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574775.618262:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574775.618266:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574775.618272:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15242 00000400:00000200:0.0:1788574775.618293:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574775.618300:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574775.618303:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574775.618315:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618319:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618323:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618327:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574775.618331:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574775.618333:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574775.618336:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574775.618338:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574775.618343:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574775.618347:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574775.618354:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618358:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618361:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618365:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574775.618369:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574775.618375:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618378:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574775.618382:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574775.618385:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574775.618390:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618394:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574775.618397:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574775.618402:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618406:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574775.618412:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574775.618416:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574775.618419:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574775.618421:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618424:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574775.618427:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574775.618430:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574775.618436:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574775.618449:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.618456:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810317568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574775.618471:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574775.618474:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.618482:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810317568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574775.618493:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574775.618499:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574775.618505:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574775.618509:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574775.618512:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574775.618518:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574775.618523:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15242 00000400:00000200:0.0:1788574775.618530:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574775.618536:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574775.618540:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574775.618546:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618551:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618553:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618557:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574775.618562:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574775.618565:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574775.618570:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574775.618573:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574775.618577:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574775.618580:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574775.618587:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618591:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618601:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618606:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574775.618611:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574775.618616:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618620:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574775.618622:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574775.618625:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574775.618628:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618633:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574775.618679:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574775.618686:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618690:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574775.618696:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574775.618701:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574775.618708:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574775.618710:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618713:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574775.618717:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574775.618731:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574775.618737:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574775.618744:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.618754:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810317696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574775.618765:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574775.618767:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.618772:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810317696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574775.618786:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574775.618791:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574775.618799:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574775.618802:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574775.618810:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574775.618815:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574775.618822:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15242 00000400:00000200:0.0:1788574775.618833:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574775.618840:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574775.618844:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574775.618851:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618856:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618858:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618861:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574775.618863:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574775.618866:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574775.618870:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574775.618872:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574775.618876:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574775.618879:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574775.618884:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618885:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574775.618888:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618894:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574775.618900:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574775.618904:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574775.618909:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574775.618914:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574775.618919:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574775.618924:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618928:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574775.618931:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574775.618936:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618941:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574775.618951:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574775.618955:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574775.618961:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574775.618965:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.618970:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574775.618974:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574775.618978:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574775.618987:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574775.618994:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.619004:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810317824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574775.619028:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574775.619032:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.619039:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810317824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574775.619059:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.619067:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15252) 00000020:00000001:0.0:1788574776.513862:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574776.513866:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574776.513869:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574779.457775:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309918430, at 4309918464 00000100:00000010:0.0:1788574779.457801:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788574781.633809:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574781.633812:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574781.633815:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574784.577754:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309923550, at 4309923584 00000100:00000010:0.0:1788574784.577769:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574786.753802:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574786.753806:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574786.753809:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574789.697750:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309928670, at 4309928704 00000100:00000010:0.0:1788574789.697768:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 10000000:00000001:0.0:1788574790.465719:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574790.465731:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9100. 00000020:00000010:0.0:1788574790.465737:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574790.465741:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574790.465745:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574790.465749:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574790.465752:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788574790.465755:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574790.465759:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ac00. 20000000:00000010:0.0:1788574790.465762:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788574790.465765:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574790.465766:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574790.465771:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574790.465774:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574790.465776:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574790.465779:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788574790.465781:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574790.465783:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ac00. 20000000:00000010:0.0:1788574790.465785:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574790.465787:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788574790.465789:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9100. 10000000:01000000:0.0:1788574790.465791:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574790.465794:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574790.465795:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574790.465796:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574790.465797:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574790.465799:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574790.465801:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574790.465803:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574790.465803:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574790.465806:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574790.465807:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574790.465809:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574790.465811:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788574791.873851:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574791.873854:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574791.873857:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788574792.129742:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574792.129749:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574792.129753:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574792.129756:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574792.129757:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574792.129758:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574792.129760:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574792.129761:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574792.129763:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574792.129766:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574792.129768:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574792.129779:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574792.129783:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574792.129788:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574792.129792:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574792.129797:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574792.129806:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574792.129813:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574792.129817:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574792.129819:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574792.129826:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574792.129829:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574792.129831:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31c00. 00000020:00000001:0.0:1788574792.129835:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574792.129837:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31c00 with handle 0x1441586093168c8e to hash 00000020:00000001:0.0:1788574792.129838:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574792.129839:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574792.129842:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574792.129844:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574792.129846:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574792.129849:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574792.129873:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574792.129882:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574792.129886:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574792.129887:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574792.129892:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574792.129893:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788574792.129902:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574792.129903:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531992192 : -109714177559424 : ffff9c372d6b7480) 00000100:00000001:0.0:1788574792.129909:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574792.129913:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574792.129915:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574792.129916:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574792.129924:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574792.129931:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574792.129934:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574792.129937:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574792.129940:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574792.129943:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574792.129945:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574792.129946:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574792.129947:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574792.129949:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574792.129951:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574792.129967:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574792.129968:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574792.129970:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574792.129972:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574792.129977:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b7480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810319872:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574792.129981:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574792.129983:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574792.129984:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574792.129985:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574792.129987:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574792.129991:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574792.129993:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574792.129997:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574792.129997:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574792.129999:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574792.130001:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574792.130003:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574792.130005:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=4 00000020:00000001:0.0:1788574792.130006:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574792.130008:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00010000:0.0:1788574792.130010:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130014:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574792.130016:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130019:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574792.130023:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130028:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574792.130029:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574792.130031:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130034:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574792.130035:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574792.130037:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574792.130038:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574792.130042:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31c00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574792.130044:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574792.130045:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130048:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574792.130049:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574792.130050:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574792.130051:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=5 00000020:00000001:0.0:1788574792.130052:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574792.130053:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574792.130055:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574792.130057:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574792.130060:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574792.130062:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574792.130065:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574792.130070:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574792.130072:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574792.130073:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574792.130074:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574792.130076:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574792.130078:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574792.130079:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574792.130080:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574792.130082:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31c00 with handle 0x1441586093168c8e from hash 00010000:00000001:0.0:1788574792.130083:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574792.130084:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574792.130085:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574792.130085:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574792.130086:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574792.130087:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574792.130089:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574792.130089:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574792.130090:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574792.130091:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574792.130093:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574792.130095:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574792.130096:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574792.130097:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574792.130098:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574792.130099:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574792.130099:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574792.130100:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574792.130101:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168c8e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574792.130105:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574792.130117:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574792.130118:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574792.130120:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574792.130125:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b7480 x1875440810319872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574792.130130:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574792.130132:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574792.130133:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574792.130135:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574792.130136:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574792.130139:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574792.130140:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574792.130141:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b7480. 00000100:00000001:0.0:1788574792.130144:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574792.130145:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574792.130146:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574792.130147:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574792.130149:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000800:00000200:0.0:1788574795.009928:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574795.969740:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309934943, at 4309934976 00000100:00000010:0.0:1788574795.969756:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788574796.993882:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574796.993886:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574796.993889:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574801.473762:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309940446, at 4309940480 00000100:00000010:0.0:1788574801.473778:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788574802.113835:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574802.113839:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574802.113843:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574802.753749:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574802.753761:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574802.753764:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574802.753768:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574802.753773:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574802.753782:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15269 00000400:00000200:0.0:1788574802.753794:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574802.753823:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574802.753826:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574802.753830:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.753835:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.753838:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.753840:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574802.753845:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574802.753848:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574802.753852:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574802.753856:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574802.753859:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574802.753861:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574802.753865:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.753867:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.753869:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.753872:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574802.753876:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574802.753880:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574802.753884:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574802.753887:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574802.753908:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574802.753912:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.753916:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574802.753918:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574802.753928:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574802.753933:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574802.753938:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574802.753940:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.753944:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574802.753948:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574802.753954:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574802.753964:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574802.753980:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.753992:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810321280/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574802.754006:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574802.754011:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.754017:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810321280/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574802.754039:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574802.754042:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574802.754045:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574802.754046:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574802.754048:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574802.754050:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574802.754052:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15269 00000400:00000200:0.0:1788574802.754056:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574802.754061:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574802.754064:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574802.754068:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754070:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754071:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754073:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574802.754076:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574802.754077:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574802.754079:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574802.754082:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574802.754087:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574802.754089:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574802.754094:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754097:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754110:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754115:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574802.754120:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574802.754126:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754130:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574802.754133:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574802.754141:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574802.754146:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754149:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574802.754156:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574802.754160:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754164:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574802.754170:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574802.754174:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574802.754178:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574802.754180:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754183:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574802.754185:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574802.754187:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574802.754195:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574802.754201:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.754211:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810321408/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574802.754231:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574802.754236:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.754246:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810321408/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574802.754268:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574802.754273:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574802.754277:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574802.754281:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574802.754292:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574802.754297:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574802.754304:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15269 00000400:00000200:0.0:1788574802.754327:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574802.754334:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574802.754338:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574802.754345:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754350:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754357:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754361:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574802.754364:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574802.754365:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574802.754367:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574802.754370:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574802.754372:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574802.754375:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574802.754379:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754381:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754392:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754398:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574802.754403:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574802.754410:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754414:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574802.754417:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574802.754424:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574802.754429:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754434:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574802.754437:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574802.754440:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754442:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574802.754454:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574802.754460:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574802.754464:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574802.754466:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754468:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574802.754471:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574802.754476:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574802.754493:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574802.754503:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.754514:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810321536/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574802.754524:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574802.754527:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.754536:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810321536/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574802.754556:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574802.754562:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574802.754574:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574802.754578:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574802.754582:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574802.754588:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574802.754596:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15269 00000400:00000200:0.0:1788574802.754608:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574802.754616:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574802.754624:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574802.754633:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754698:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754704:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754710:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574802.754714:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574802.754727:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574802.754733:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574802.754737:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574802.754743:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574802.754749:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574802.754758:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754766:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.754772:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754779:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574802.754787:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574802.754795:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574802.754801:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574802.754807:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574802.754813:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574802.754820:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754826:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574802.754830:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574802.754835:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754842:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574802.754850:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574802.754856:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574802.754863:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574802.754868:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.754875:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574802.754878:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574802.754881:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574802.754886:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574802.754890:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.754908:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810321664/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574802.754921:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574802.754926:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.754942:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810321664/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574802.754959:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574802.754962:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574802.754972:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574802.754976:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574802.754979:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574802.754984:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574802.754991:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15269 00000400:00000200:0.0:1788574802.755000:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574802.755008:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574802.755014:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574802.755021:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.755026:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.755032:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.755036:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574802.755043:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574802.755046:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574802.755051:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574802.755056:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574802.755061:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574802.755067:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574802.755073:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.755078:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574802.755083:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574802.755087:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574802.755092:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574802.755100:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574802.755107:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574802.755118:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574802.755124:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574802.755129:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.755134:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574802.755138:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574802.755143:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.755148:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574802.755157:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574802.755163:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574802.755170:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574802.755174:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574802.755179:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574802.755182:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574802.755184:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574802.755193:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574802.755200:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.755211:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810321792/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574802.755236:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574802.755240:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.755247:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810321792/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574802.755267:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574802.755276:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15279) 00000100:00000040:0.0:1788574806.593809:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309945566, at 4309945600 00000100:00000010:0.0:1788574806.593813:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788574807.233888:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574807.233892:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574807.233896:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574807.873752:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574807.873762:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574807.873764:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574807.873779:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574807.873783:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574807.873790:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15274 00000400:00000200:0.0:1788574807.873799:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574807.873805:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574807.873808:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574807.873812:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.873815:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.873818:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.873820:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574807.873824:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574807.873826:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574807.873828:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574807.873831:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574807.873834:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574807.873836:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574807.873839:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.873841:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.873842:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.873844:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574807.873846:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574807.873848:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574807.873850:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574807.873853:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574807.873855:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574807.873857:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.873858:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574807.873860:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574807.873865:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574807.873868:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574807.873870:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574807.873875:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.873877:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574807.873880:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574807.873884:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574807.873890:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574807.873902:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.873912:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810321920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574807.873922:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574807.873926:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.873931:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810321920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574807.873957:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574807.873960:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574807.873963:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574807.873964:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574807.873966:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574807.873980:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574807.873983:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15274 00000400:00000200:0.0:1788574807.873987:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574807.873991:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574807.873993:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574807.873996:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874000:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874002:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874004:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574807.874006:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574807.874007:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574807.874009:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574807.874012:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574807.874014:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574807.874021:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574807.874025:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874027:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874029:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874031:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574807.874035:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574807.874038:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874040:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574807.874043:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574807.874050:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574807.874053:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874055:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574807.874061:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574807.874067:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874070:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574807.874077:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574807.874082:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574807.874085:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574807.874086:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874087:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574807.874089:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574807.874093:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574807.874098:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574807.874103:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.874112:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810322048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574807.874127:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574807.874131:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.874138:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810322048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574807.874149:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574807.874159:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574807.874166:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574807.874169:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574807.874171:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574807.874177:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574807.874180:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15274 00000400:00000200:0.0:1788574807.874187:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574807.874197:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574807.874201:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574807.874207:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874210:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874215:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874220:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574807.874225:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574807.874227:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574807.874230:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574807.874234:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574807.874238:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574807.874241:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574807.874248:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874252:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874254:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874256:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574807.874261:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574807.874265:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874268:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574807.874271:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574807.874276:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574807.874279:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874283:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574807.874294:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574807.874299:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874303:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574807.874307:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574807.874310:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574807.874313:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574807.874316:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874318:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574807.874319:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574807.874323:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574807.874330:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574807.874336:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.874344:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810322176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574807.874355:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574807.874359:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.874367:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810322176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574807.874379:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574807.874383:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574807.874387:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574807.874388:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574807.874389:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574807.874391:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574807.874395:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15274 00000400:00000200:0.0:1788574807.874401:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574807.874406:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574807.874410:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574807.874428:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874432:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874435:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874442:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574807.874448:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574807.874450:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574807.874453:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574807.874457:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574807.874461:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574807.874466:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574807.874474:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874478:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874483:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874486:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574807.874490:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574807.874495:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874499:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574807.874503:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574807.874508:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574807.874513:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874518:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574807.874522:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574807.874526:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874530:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574807.874534:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574807.874539:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574807.874544:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574807.874547:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874551:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574807.874554:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574807.874558:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574807.874565:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574807.874576:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.874584:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810322304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574807.874605:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574807.874610:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.874615:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810322304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574807.874627:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574807.874632:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574807.874677:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574807.874680:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574807.874683:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574807.874689:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574807.874695:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15274 00000400:00000200:0.0:1788574807.874702:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574807.874710:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574807.874714:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574807.874720:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874724:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874728:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874733:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574807.874749:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574807.874752:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574807.874754:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574807.874758:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574807.874763:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574807.874767:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574807.874775:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874780:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574807.874785:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874790:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574807.874800:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574807.874808:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574807.874812:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574807.874818:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574807.874824:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574807.874829:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874833:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574807.874835:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574807.874840:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874843:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574807.874849:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574807.874853:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574807.874858:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574807.874861:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.874862:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574807.874864:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574807.874869:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574807.874873:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574807.874879:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.874886:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810322432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574807.874902:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574807.874906:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.874915:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810322432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574807.874928:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574807.874936:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15284) 00000100:00000040:0.0:1788574811.713740:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309950686, at 4309950720 00000100:00000010:0.0:1788574811.713755:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574812.353944:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574812.353948:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574812.353951:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574817.473867:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574817.473870:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574817.473873:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574818.369744:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309957343, at 4309957376 00000100:00000010:0.0:1788574818.369758:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788574822.593836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574822.593839:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574822.593842:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574823.489708:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309962462, at 4309962496 00000100:00000010:0.0:1788574823.489724:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788574827.713848:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574827.713851:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574827.713854:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574828.609735:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309967582, at 4309967616 00000100:00000010:0.0:1788574828.609750:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788574832.833933:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574832.833936:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574832.833938:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574833.729748:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309972702, at 4309972736 00000100:00000010:0.0:1788574833.729766:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000800:00000200:0.0:1788574833.921937:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 10000000:00000001:0.0:1788574835.009740:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574835.009753:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a97c0. 00000020:00000010:0.0:1788574835.009762:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574835.009768:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574835.009774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574835.009778:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574835.009784:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788574835.009788:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574835.009793:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788574835.009796:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788574835.009800:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574835.009803:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574835.009819:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574835.009824:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574835.009828:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574835.009832:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788574835.009837:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574835.009842:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788574835.009846:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574835.009849:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788574835.009852:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a97c0. 10000000:01000000:0.0:1788574835.009855:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574835.009859:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574835.009861:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574835.009863:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574835.009866:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574835.009870:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574835.009872:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574835.009877:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574835.009879:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574835.009881:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574835.009885:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574835.009888:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574835.009891:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574836.929734:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574836.929740:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574836.929745:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574836.929749:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574836.929751:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574836.929752:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574836.929754:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574836.929758:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574836.929762:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574836.929765:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574836.929769:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574836.929780:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574836.929785:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574836.929788:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574836.929802:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574836.929806:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574836.929812:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574836.929817:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574836.929821:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574836.929823:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574836.929828:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574836.929831:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574836.929833:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788574836.929836:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574836.929838:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x1441586093168c9c to hash 00000020:00000001:0.0:1788574836.929840:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574836.929843:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574836.929845:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574836.929850:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574836.929855:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574836.929859:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574836.929866:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574836.929874:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574836.929878:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574836.929879:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574836.929884:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574836.929885:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788574836.929887:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574836.929888:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788574836.929891:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574836.929894:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574836.929896:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574836.929900:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574836.929909:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574836.929918:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574836.929922:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574836.929928:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574836.929932:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574836.929939:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574836.929942:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574836.929943:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574836.929946:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574836.929948:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574836.929951:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574836.929953:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574836.929955:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574836.929956:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574836.929959:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574836.929964:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810326528:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574836.929968:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574836.929969:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574836.929971:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574836.929972:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574836.929974:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574836.929978:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574836.929980:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574836.930007:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574836.930011:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574836.930013:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574836.930016:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574836.930017:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574836.930019:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=4 00000020:00000001:0.0:1788574836.930021:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574836.930023:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00010000:0.0:1788574836.930027:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930033:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574836.930036:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930042:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574836.930045:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930051:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574836.930053:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574836.930055:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930061:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574836.930063:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574836.930065:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574836.930067:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574836.930071:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31e00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574836.930072:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574836.930074:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930088:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574836.930089:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574836.930090:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574836.930092:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=5 00000020:00000001:0.0:1788574836.930093:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574836.930096:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574836.930098:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574836.930101:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574836.930107:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574836.930109:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574836.930114:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574836.930117:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574836.930118:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574836.930120:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574836.930122:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574836.930124:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574836.930126:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574836.930127:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574836.930129:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574836.930131:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31e00 with handle 0x1441586093168c9c from hash 00010000:00000001:0.0:1788574836.930133:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574836.930134:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574836.930135:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574836.930136:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574836.930137:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574836.930139:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574836.930141:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574836.930142:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574836.930143:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574836.930149:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574836.930153:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574836.930158:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574836.930159:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574836.930160:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574836.930162:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574836.930164:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574836.930165:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574836.930166:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574836.930168:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168c9c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574836.930175:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574836.930183:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574836.930185:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574836.930187:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574836.930191:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810326528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574836.930198:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574836.930200:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574836.930202:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574836.930204:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574836.930206:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574836.930210:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574836.930212:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574836.930214:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788574836.930217:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574836.930218:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574836.930220:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574836.930223:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574836.930225:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574837.953819:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574837.953823:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574837.953826:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574840.769749:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309979743, at 4309979776 00000100:00000010:0.0:1788574840.769767:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788574843.073921:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574843.073924:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574843.073926:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574846.017707:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309984990, at 4309985024 00000100:00000010:0.0:1788574846.017722:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788574848.193866:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574848.193869:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574848.193871:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574851.137748:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309990110, at 4309990144 00000100:00000010:0.0:1788574851.137765:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788574853.313892:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574853.313898:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574853.313903:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574856.257751:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4309995231, at 4309995264 00000100:00000010:0.0:1788574856.257769:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 10000000:00000001:0.0:1788574857.025748:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574857.025762:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9c00. 00000020:00000010:0.0:1788574857.025774:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574857.025781:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788574857.025787:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574857.025793:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574857.025802:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574857.025807:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574857.025813:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788574857.025819:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788574857.025825:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574857.025828:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574857.025835:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788574857.025842:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574857.025846:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574857.025865:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574857.025870:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574857.025874:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788574857.025879:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574857.025883:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788574857.025888:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9c00. 10000000:01000000:0.0:1788574857.025892:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574857.025897:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574857.025900:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574857.025903:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574857.025906:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574857.025910:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574857.025913:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574857.025917:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574857.025919:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574857.025922:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574857.025925:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574857.025927:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574857.025930:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574858.369768:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574858.369774:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574858.369779:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574858.369781:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574858.369783:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574858.369785:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574858.369787:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574858.369789:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574858.369792:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574858.369794:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574858.369796:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574858.369806:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574858.369810:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574858.369814:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574858.369818:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574858.369823:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574858.369829:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788574858.369835:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574858.369852:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574858.369854:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574858.369861:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574858.369864:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574858.369867:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788574858.369870:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574858.369873:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168ca3 to hash 00000020:00000001:0.0:1788574858.369874:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574858.369875:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574858.369877:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574858.369880:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574858.369883:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574858.369886:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574858.369894:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574858.369903:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574858.369908:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574858.369909:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574858.369914:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574858.369915:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788574858.369917:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.369919:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788574858.369930:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574858.369934:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.369935:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574858.369937:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.369945:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369952:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574858.369959:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369964:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574858.369967:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369971:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574858.369973:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574858.369974:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.369976:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574858.369978:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574858.369980:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574858.369982:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.369984:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.369985:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.369988:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.369994:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810329856:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574858.369999:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370001:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574858.370002:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574858.370004:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370006:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370011:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574858.370012:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788574858.370016:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574858.370017:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574858.370020:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574858.370022:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574858.370024:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574858.370030:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788574858.370031:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574858.370033:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788574858.370036:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370041:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574858.370044:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370047:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574858.370049:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370053:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574858.370055:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574858.370056:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370060:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574858.370061:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574858.370063:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574858.370065:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574858.370069:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574858.370071:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574858.370072:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370076:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574858.370077:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574858.370080:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574858.370081:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788574858.370083:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574858.370084:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574858.370087:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574858.370089:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574858.370092:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574858.370094:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574858.370097:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574858.370099:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574858.370100:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574858.370102:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574858.370103:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574858.370105:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574858.370107:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574858.370108:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574858.370109:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574858.370110:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168ca3 from hash 00010000:00000001:0.0:1788574858.370112:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574858.370113:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574858.370113:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574858.370114:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574858.370115:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574858.370116:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574858.370118:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574858.370118:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574858.370120:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574858.370121:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574858.370123:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574858.370133:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574858.370133:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574858.370134:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574858.370136:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574858.370137:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574858.370137:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574858.370138:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574858.370140:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168ca3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574858.370144:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574858.370152:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574858.370153:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574858.370155:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370157:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810329856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370162:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574858.370164:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.370165:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574858.370166:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.370168:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788574858.370171:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.370173:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.370174:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788574858.370177:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.370178:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574858.370179:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574858.370181:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574858.370183:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574858.433824:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574858.433827:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574858.433830:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574862.209793:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310001183, at 4310001216 00000100:00000010:0.0:1788574862.209824:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788574863.553982:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574863.553986:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574863.553989:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574867.521731:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310006495, at 4310006528 00000100:00000010:0.0:1788574867.521750:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788574868.673808:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574868.673814:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574868.673817:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788574871.809917:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788574872.641758:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310011615, at 4310011648 00000100:00000010:0.0:1788574872.641774:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788574873.793844:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574873.793847:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574873.793851:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574877.761717:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310016734, at 4310016768 00000100:00000010:0.0:1788574877.761739:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 10000000:00000001:0.0:1788574878.529743:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574878.529755:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9600. 00000020:00000010:0.0:1788574878.529765:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574878.529773:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788574878.529777:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574878.529783:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574878.529791:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574878.529796:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574878.529802:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574878.529808:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788574878.529813:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574878.529816:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574878.529822:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788574878.529831:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574878.529834:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574878.529838:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574878.529857:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574878.529860:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574878.529862:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574878.529864:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788574878.529866:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9600. 10000000:01000000:0.0:1788574878.529869:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574878.529871:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574878.529873:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574878.529875:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574878.529876:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574878.529879:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574878.529880:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574878.529883:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574878.529884:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574878.529887:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574878.529891:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574878.529893:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574878.529895:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788574878.913894:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574878.913906:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574878.913911:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788574879.617718:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574879.617724:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574879.617727:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574879.617729:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574879.617730:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574879.617731:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574879.617734:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574879.617736:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574879.617738:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574879.617741:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574879.617742:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574879.617751:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574879.617754:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574879.617758:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574879.617761:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574879.617765:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574879.617781:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574879.617786:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574879.617790:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574879.617792:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574879.617798:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574879.617801:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574879.617804:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788574879.617807:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574879.617809:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168caa to hash 00000020:00000001:0.0:1788574879.617810:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574879.617811:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574879.617814:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574879.617816:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574879.617818:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574879.617821:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574879.617828:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574879.617835:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574879.617839:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574879.617840:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574879.617844:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574879.617846:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788574879.617848:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574879.617849:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788574879.617852:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574879.617855:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574879.617857:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574879.617858:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574879.617865:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574879.617874:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574879.617877:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574879.617881:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574879.617883:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574879.617887:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574879.617889:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574879.617890:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574879.617891:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574879.617893:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574879.617894:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574879.617897:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574879.617898:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574879.617899:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574879.617902:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574879.617906:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810333184:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574879.617910:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574879.617911:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574879.617913:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574879.617914:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574879.617916:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574879.617920:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574879.617921:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574879.617925:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574879.617926:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574879.617927:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574879.617932:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574879.617933:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574879.617935:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788574879.617936:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574879.617938:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788574879.617940:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.617944:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574879.617946:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.617950:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574879.617952:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.617967:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574879.617969:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574879.617971:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.617975:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574879.617976:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574879.617978:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574879.617980:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574879.617984:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574879.617985:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574879.617987:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.617996:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574879.617997:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574879.617998:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574879.617999:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788574879.618000:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574879.618001:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788574879.618002:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574879.618004:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574879.618007:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574879.618008:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574879.618011:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574879.618013:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574879.618014:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574879.618016:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574879.618017:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574879.618019:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574879.618020:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574879.618021:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574879.618022:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574879.618024:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168caa from hash 00010000:00000001:0.0:1788574879.618025:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574879.618026:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574879.618026:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574879.618027:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574879.618028:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574879.618029:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574879.618030:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574879.618031:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574879.618032:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574879.618035:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574879.618037:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574879.618040:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574879.618041:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574879.618042:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574879.618043:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574879.618044:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574879.618044:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574879.618045:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574879.618046:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168caa lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574879.618050:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574879.618056:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574879.618057:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574879.618058:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574879.618061:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810333184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574879.618064:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574879.618066:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574879.618067:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574879.618069:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574879.618070:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574879.618073:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574879.618081:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574879.618082:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788574879.618085:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574879.618088:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574879.618090:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574879.618091:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574879.618093:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788574883.457733:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310022431, at 4310022464 00000100:00000010:0.0:1788574883.457748:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788574884.033833:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574884.033837:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574884.033839:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574888.513755:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310027486, at 4310027520 00000100:00000010:0.0:1788574888.513771:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788574889.153870:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574889.153873:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574889.153877:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574893.633719:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310032606, at 4310032640 00000100:00000010:0.0:1788574893.633739:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788574894.273819:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574894.273822:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574894.273825:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574898.753714:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310037727, at 4310037760 00000100:00000010:0.0:1788574898.753725:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788574899.393836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574899.393840:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574899.393843:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788574900.033751:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574900.033760:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9600. 00000020:00000010:0.0:1788574900.033767:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574900.033774:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574900.033780:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574900.033788:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574900.033797:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574900.033817:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574900.033825:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574900.033834:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788574900.033839:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574900.033842:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574900.033850:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574900.033859:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574900.033867:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574900.033883:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574900.033890:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574900.033895:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574900.033902:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574900.033907:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788574900.033913:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9600. 10000000:01000000:0.0:1788574900.033920:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574900.033926:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574900.033931:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574900.033935:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574900.033939:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574900.033942:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574900.033945:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574900.033950:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574900.033961:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574900.033965:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574900.033970:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574900.033972:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574900.033977:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574901.249781:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574901.249803:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574901.249808:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574901.249812:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574901.249813:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574901.249814:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574901.249819:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574901.249821:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574901.249823:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574901.249826:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574901.249827:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574901.249838:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574901.249842:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574901.249845:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574901.249848:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574901.249852:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574901.249859:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574901.249864:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574901.249879:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574901.249883:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574901.249888:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788574901.249892:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574901.249894:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788574901.249898:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574901.249901:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x1441586093168cb1 to hash 00000020:00000001:0.0:1788574901.249902:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574901.249903:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574901.249906:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574901.249909:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574901.249911:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574901.249914:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574901.249922:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574901.249931:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574901.249936:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574901.249937:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574901.249943:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574901.249944:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788574901.249946:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574901.249947:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788574901.249951:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574901.249955:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574901.249957:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574901.249959:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574901.249968:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574901.249974:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574901.249980:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574901.249985:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574901.249988:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574901.249992:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574901.249994:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574901.249995:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574901.249997:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574901.249999:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574901.250001:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574901.250003:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.250005:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574901.250007:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574901.250009:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574901.250015:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810336512:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574901.250019:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.250021:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574901.250023:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574901.250024:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574901.250027:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574901.250031:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574901.250033:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574901.250037:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574901.250038:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574901.250040:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574901.250042:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574901.250044:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574901.250048:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=4 00000020:00000001:0.0:1788574901.250050:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574901.250052:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00010000:0.0:1788574901.250054:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250058:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574901.250061:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250065:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574901.250067:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250070:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574901.250072:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574901.250074:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250077:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574901.250078:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574901.250081:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574901.250083:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574901.250087:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31e00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574901.250089:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574901.250091:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250094:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574901.250095:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574901.250099:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574901.250100:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=5 00000020:00000001:0.0:1788574901.250101:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574901.250103:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788574901.250105:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574901.250107:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574901.250111:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574901.250113:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574901.250116:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574901.250118:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574901.250119:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574901.250121:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574901.250122:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574901.250125:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574901.250126:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574901.250127:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574901.250128:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574901.250129:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31e00 with handle 0x1441586093168cb1 from hash 00010000:00000001:0.0:1788574901.250131:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574901.250132:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574901.250133:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574901.250133:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574901.250135:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574901.250136:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574901.250138:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574901.250138:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574901.250139:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574901.250140:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574901.250142:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574901.250151:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574901.250152:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574901.250153:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574901.250155:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574901.250156:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574901.250156:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574901.250157:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574901.250159:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168cb1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574901.250163:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788574901.250177:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574901.250178:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574901.250180:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574901.250183:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810336512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574901.250187:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574901.250189:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574901.250190:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574901.250191:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574901.250194:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574901.250197:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574901.250199:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574901.250200:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788574901.250203:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574901.250204:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574901.250205:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574901.250207:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574901.250209:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574904.513864:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574904.513867:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574904.513870:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574905.089738:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310044063, at 4310044096 00000100:00000010:0.0:1788574905.089754:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215960. 00000020:00000001:0.0:1788574909.633946:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574909.633951:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574909.633955:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574910.529774:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310049502, at 4310049536 00000100:00000010:0.0:1788574910.529792:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000800:00000200:0.0:1788574910.722076:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574914.753862:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574914.753868:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574914.753872:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574915.649772:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310054622, at 4310054656 00000100:00000010:0.0:1788574915.649788:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00080000:0.0:1788574916.929802:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574916.929811:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574916.929813:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574916.929816:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574916.929827:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574916.929834:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15383 00000400:00000200:0.0:1788574916.929841:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574916.929846:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574916.929849:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574916.929851:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.929854:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.929856:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.929857:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574916.929860:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574916.929862:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574916.929864:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574916.929866:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574916.929867:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574916.929869:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574916.929871:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.929887:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.929888:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.929890:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574916.929892:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574916.929895:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574916.929896:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574916.929898:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574916.929901:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574916.929903:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.929904:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574916.929906:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574916.929911:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574916.929913:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574916.929916:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574916.929917:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.929920:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574916.929922:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574916.929926:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574916.929932:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574916.929938:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.929945:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810338560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574916.929952:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574916.929955:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.929958:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810338560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574916.929975:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574916.929978:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574916.929980:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574916.929981:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574916.929983:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574916.929985:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574916.929990:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15383 00000400:00000200:0.0:1788574916.929993:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574916.929995:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574916.929997:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574916.929999:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930000:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930002:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930004:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574916.930006:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574916.930007:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574916.930009:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574916.930010:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574916.930011:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574916.930013:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574916.930015:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930016:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930018:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930019:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574916.930024:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574916.930026:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930032:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574916.930036:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574916.930043:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574916.930049:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930053:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574916.930059:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574916.930064:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930076:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574916.930083:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574916.930090:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574916.930096:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574916.930105:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930109:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574916.930114:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574916.930119:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574916.930126:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574916.930132:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.930139:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810338688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574916.930147:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574916.930150:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.930154:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810338688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574916.930161:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574916.930163:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574916.930165:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574916.930167:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574916.930168:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574916.930169:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574916.930171:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15383 00000400:00000200:0.0:1788574916.930174:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574916.930176:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574916.930178:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574916.930180:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930181:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930182:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930184:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574916.930185:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574916.930186:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574916.930187:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574916.930188:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574916.930190:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574916.930196:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574916.930198:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930199:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930201:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930202:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574916.930203:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574916.930205:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930206:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574916.930207:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574916.930210:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574916.930211:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930213:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574916.930222:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574916.930224:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930225:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574916.930229:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574916.930232:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574916.930235:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574916.930238:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930247:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574916.930248:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574916.930250:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574916.930253:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574916.930262:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.930270:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810338816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574916.930276:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574916.930277:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.930279:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810338816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574916.930285:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574916.930296:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574916.930303:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574916.930307:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574916.930308:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574916.930314:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574916.930317:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15383 00000400:00000200:0.0:1788574916.930320:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574916.930324:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574916.930328:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574916.930337:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930344:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930354:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930363:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574916.930372:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574916.930380:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574916.930386:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574916.930389:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574916.930398:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574916.930406:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574916.930418:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930427:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930432:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930445:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574916.930454:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574916.930466:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930474:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574916.930483:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574916.930492:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574916.930499:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930504:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574916.930514:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574916.930520:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930522:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574916.930525:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574916.930529:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574916.930536:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574916.930538:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930543:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574916.930548:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574916.930555:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574916.930561:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574916.930564:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.930572:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810338944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574916.930580:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574916.930584:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.930587:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810338944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574916.930608:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574916.930612:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574916.930625:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574916.930628:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574916.930634:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574916.930686:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574916.930689:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15383 00000400:00000200:0.0:1788574916.930694:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574916.930697:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574916.930706:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574916.930716:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930725:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930735:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930753:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574916.930760:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574916.930761:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574916.930763:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574916.930766:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574916.930772:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574916.930774:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574916.930778:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930783:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574916.930791:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930794:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574916.930799:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574916.930805:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574916.930808:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574916.930813:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574916.930818:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574916.930824:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930829:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574916.930831:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574916.930834:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930839:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574916.930848:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574916.930857:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574916.930866:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574916.930870:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574916.930878:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574916.930882:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574916.930887:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574916.930892:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574916.930896:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.930904:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810339072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574916.930932:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574916.930937:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.930943:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810339072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574916.930970:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574916.930982:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15393) 00000020:00000001:0.0:1788574919.873808:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574919.873811:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574919.873814:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574920.769728:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310059743, at 4310059776 00000100:00000010:0.0:1788574920.769742:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 10000000:00000001:0.0:1788574921.537729:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574921.537771:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e40. 00000020:00000010:0.0:1788574921.537778:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574921.537782:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788574921.537785:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574921.537789:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574921.537794:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574921.537798:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574921.537803:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574921.537806:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788574921.537810:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574921.537813:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574921.537817:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788574921.537835:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788574921.537838:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574921.537842:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574921.537846:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574921.537850:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788574921.537853:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788574921.537857:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788574921.537873:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e40. 10000000:01000000:0.0:1788574921.537876:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574921.537880:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574921.537882:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574921.537884:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574921.537886:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574921.537889:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574921.537891:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574921.537893:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574921.537895:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574921.537897:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574921.537899:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574921.537901:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574921.537903:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574923.009782:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574923.009790:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574923.009794:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574923.009797:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574923.009798:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574923.009800:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574923.009803:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574923.009805:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574923.009808:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574923.009810:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574923.009812:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574923.009823:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574923.009827:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574923.009832:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.009836:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.009841:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.009847:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574923.009854:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574923.009857:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574923.009860:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574923.009866:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692b00. 00010000:00000001:0.0:1788574923.009870:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574923.009872:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788574923.009888:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574923.009890:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168cb8 to hash 00000020:00000001:0.0:1788574923.009892:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574923.009893:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574923.009895:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574923.009898:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574923.009901:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574923.009904:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574923.009912:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574923.009920:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574923.009925:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574923.009926:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574923.009931:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574923.009932:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788574923.009934:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574923.009935:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788574923.009939:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.009942:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.009944:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574923.009945:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574923.009953:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.009960:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574923.009962:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.009967:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.009973:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.009978:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574923.009980:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574923.009981:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.009983:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574923.009985:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574923.009986:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574923.009989:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574923.009990:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574923.009992:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574923.009995:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574923.010000:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810339840:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574923.010004:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574923.010006:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574923.010007:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574923.010009:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574923.010012:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.010016:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.010018:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574923.010022:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574923.010023:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574923.010024:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574923.010027:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574923.010028:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010030:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788574923.010031:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574923.010033:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788574923.010036:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010043:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574923.010046:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010051:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574923.010053:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010056:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574923.010057:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574923.010059:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010063:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574923.010064:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574923.010066:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574923.010068:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574923.010072:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574923.010073:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574923.010075:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010078:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574923.010079:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574923.010080:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010081:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788574923.010082:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574923.010084:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788574923.010089:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574923.010091:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574923.010096:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574923.010097:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574923.010101:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574923.010103:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574923.010104:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574923.010105:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574923.010107:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574923.010109:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574923.010110:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574923.010111:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574923.010112:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574923.010114:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168cb8 from hash 00010000:00000001:0.0:1788574923.010115:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574923.010116:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574923.010117:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574923.010118:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574923.010119:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574923.010120:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574923.010121:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574923.010122:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574923.010123:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574923.010124:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574923.010126:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574923.010130:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574923.010131:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574923.010132:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574923.010133:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574923.010139:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574923.010140:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574923.010141:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574923.010143:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168cb8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574923.010148:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692b00 count: 0 00010000:00000001:0.0:1788574923.010156:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574923.010157:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574923.010159:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574923.010161:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810339840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.010166:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574923.010168:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574923.010169:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574923.010171:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574923.010173:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574923.010176:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574923.010177:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574923.010178:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788574923.010180:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574923.010181:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574923.010183:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574923.010184:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574923.010187:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:0.0:1788574923.010258:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574923.010263:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574923.010265:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574923.010267:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574923.010271:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574923.010274:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15393 00000400:00000200:0.0:1788574923.010282:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574923.010287:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574923.010289:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574923.010296:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010298:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010300:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010302:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574923.010304:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574923.010305:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574923.010306:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010308:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574923.010309:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574923.010311:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574923.010312:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010314:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010315:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010316:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574923.010318:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574923.010320:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010322:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574923.010324:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574923.010327:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574923.010328:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010330:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574923.010331:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.010333:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574923.010335:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574923.010336:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574923.010338:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010339:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.010341:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.010342:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.010346:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574923.010348:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.010352:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810339968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574923.010361:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.010363:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.010366:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810339968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.010382:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574923.010384:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574923.010387:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574923.010388:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574923.010390:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574923.010391:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574923.010393:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15393 00000400:00000200:0.0:1788574923.010396:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574923.010398:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574923.010399:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574923.010401:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010402:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010404:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010405:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574923.010407:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574923.010408:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574923.010409:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010410:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574923.010411:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574923.010412:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574923.010414:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010415:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010417:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010418:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574923.010423:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574923.010426:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010428:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574923.010429:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574923.010434:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574923.010436:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010437:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574923.010441:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574923.010445:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010447:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.010452:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574923.010456:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574923.010458:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574923.010461:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010463:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.010466:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.010467:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.010472:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574923.010477:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.010485:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810340096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574923.010496:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.010499:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.010506:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810340096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.010519:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574923.010524:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574923.010528:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574923.010531:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574923.010532:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574923.010536:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574923.010542:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15393 00000400:00000200:0.0:1788574923.010545:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574923.010548:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574923.010552:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574923.010554:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010566:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010571:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010573:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574923.010575:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574923.010577:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574923.010580:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010583:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574923.010584:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574923.010586:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574923.010592:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010593:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010596:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010602:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574923.010604:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574923.010606:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010608:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574923.010614:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574923.010620:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574923.010684:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010690:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574923.010693:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574923.010697:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010702:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.010708:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574923.010712:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574923.010715:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574923.010722:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010724:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.010727:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.010730:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.010735:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574923.010741:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.010750:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810340224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574923.010758:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.010762:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.010769:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810340224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.010776:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574923.010779:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574923.010783:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574923.010787:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574923.010788:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574923.010792:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574923.010798:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15393 00000400:00000200:0.0:1788574923.010804:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574923.010810:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574923.010814:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574923.010820:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010825:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010830:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010834:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574923.010838:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574923.010841:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574923.010845:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.010850:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574923.010855:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574923.010866:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574923.010874:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010880:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.010885:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010890:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574923.010896:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574923.010904:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574923.010908:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574923.010910:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574923.010915:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574923.010922:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010927:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574923.010930:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574923.010936:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010939:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.010947:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574923.010953:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574923.010958:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574923.010961:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.010965:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.010969:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.010972:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.010980:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574923.010988:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.010999:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810340352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574923.011024:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.011029:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.011039:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810340352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.011058:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574923.011072:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574923.011079:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574923.011083:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574923.011086:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574923.011089:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574923.011094:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15393 00000400:00000200:0.0:1788574923.011101:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574923.011106:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574923.011108:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574923.011114:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.011118:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.011122:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.011126:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574923.011131:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574923.011134:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574923.011135:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574923.011137:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574923.011141:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574923.011145:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574923.011148:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.011152:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574923.011154:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574923.011159:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574923.011161:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574923.011166:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574923.011168:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574923.011172:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574923.011175:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574923.011177:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.011181:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574923.011187:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574923.011190:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.011192:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574923.011196:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574923.011201:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574923.011206:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574923.011210:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574923.011213:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574923.011216:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574923.011220:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574923.011228:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574923.011235:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.011246:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810340480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574923.011267:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574923.011272:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.011282:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810340480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574923.011309:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.011317:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15399) 00000020:00000001:0.0:1788574924.993827:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574924.993833:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574924.993837:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574926.849756:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310065823, at 4310065856 00000100:00000010:0.0:1788574926.849776:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215ae0. 00000100:00080000:0.0:1788574928.193715:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574928.193723:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574928.193725:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574928.193728:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574928.193732:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574928.193738:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15394 00000400:00000200:0.0:1788574928.193745:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574928.193765:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574928.193768:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574928.193770:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193773:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.193775:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193777:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574928.193780:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574928.193781:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574928.193783:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574928.193786:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574928.193787:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574928.193789:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574928.193791:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193793:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.193794:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193795:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574928.193798:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574928.193800:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193801:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574928.193803:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574928.193806:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574928.193808:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.193809:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574928.193811:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574928.193816:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574928.193818:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574928.193821:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574928.193822:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.193825:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574928.193827:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574928.193830:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574928.193837:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574928.193845:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.193853:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810340608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574928.193860:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574928.193864:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.193867:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810340608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574928.193885:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574928.193887:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574928.193890:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574928.193891:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574928.193892:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574928.193894:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574928.193896:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15394 00000400:00000200:0.0:1788574928.193898:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574928.193900:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574928.193902:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574928.193904:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193906:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.193908:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193910:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574928.193912:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574928.193913:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574928.193914:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574928.193916:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574928.193917:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574928.193919:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574928.193921:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193922:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.193929:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193931:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574928.193934:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574928.193937:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574928.193938:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574928.193939:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574928.193944:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574928.193947:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.193949:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574928.193955:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574928.193959:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.193961:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574928.193967:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574928.193970:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574928.193975:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574928.193977:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.193981:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574928.193984:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574928.193986:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574928.193991:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574928.193997:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.194003:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810340736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574928.194015:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574928.194019:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.194024:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810340736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574928.194036:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574928.194039:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574928.194045:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574928.194048:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574928.194055:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574928.194059:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574928.194062:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15394 00000400:00000200:0.0:1788574928.194068:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574928.194071:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574928.194072:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574928.194074:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194075:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194077:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194083:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574928.194084:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574928.194085:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574928.194087:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574928.194091:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574928.194095:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574928.194097:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574928.194100:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194105:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194107:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194108:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574928.194110:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574928.194112:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194117:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574928.194119:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574928.194121:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574928.194123:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194127:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574928.194131:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574928.194137:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194141:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574928.194153:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574928.194159:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574928.194164:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574928.194169:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194174:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574928.194179:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574928.194184:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574928.194190:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574928.194197:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.194206:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810340864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574928.194217:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574928.194220:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.194225:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810340864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574928.194234:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574928.194237:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574928.194242:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574928.194246:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574928.194247:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574928.194251:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574928.194254:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15394 00000400:00000200:0.0:1788574928.194259:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574928.194267:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574928.194273:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574928.194277:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194281:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194287:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194290:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574928.194293:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574928.194302:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574928.194304:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574928.194306:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574928.194310:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574928.194315:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574928.194321:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194326:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194328:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194330:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574928.194335:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574928.194340:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194342:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574928.194344:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574928.194347:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574928.194350:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194352:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574928.194355:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574928.194360:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194362:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574928.194370:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574928.194374:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574928.194379:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574928.194380:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194384:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574928.194386:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574928.194388:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574928.194396:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574928.194402:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.194409:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810340992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574928.194421:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574928.194424:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.194434:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810340992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574928.194446:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574928.194449:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574928.194454:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574928.194456:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574928.194459:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574928.194462:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574928.194467:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15394 00000400:00000200:0.0:1788574928.194472:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574928.194476:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574928.194479:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574928.194484:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194489:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194493:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194498:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574928.194501:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574928.194503:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574928.194505:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574928.194509:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574928.194511:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574928.194514:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574928.194518:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194522:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574928.194526:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194530:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574928.194535:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574928.194540:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574928.194543:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574928.194549:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574928.194552:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574928.194554:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194556:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574928.194559:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574928.194562:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194567:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574928.194572:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574928.194575:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574928.194581:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574928.194608:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574928.194612:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574928.194615:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574928.194616:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574928.194621:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574928.194624:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.194628:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810341120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574928.194676:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574928.194678:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.194685:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810341120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574928.194699:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.194707:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15404) 00000020:00000001:0.0:1788574930.113841:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574930.113844:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574930.113847:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574932.033738:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310071006, at 4310071040 00000100:00000010:0.0:1788574932.033752:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215ba0. 00000020:00000001:0.0:1788574935.233831:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574935.233835:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574935.233838:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574937.153753:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310076126, at 4310076160 00000100:00000010:0.0:1788574937.153783:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788574940.353804:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574940.353807:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574940.353809:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574942.273757:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310081246, at 4310081280 00000100:00000010:0.0:1788574942.273775:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215cc0. 10000000:00000001:0.0:1788574943.041762:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788574943.041774:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e40. 00000020:00000010:0.0:1788574943.041780:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788574943.041784:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574943.041787:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574943.041790:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574943.041794:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574943.041797:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574943.041800:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574943.041804:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4be00. 10000000:00000040:0.0:1788574943.041807:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788574943.041809:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788574943.041813:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788574943.041818:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788574943.041820:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788574943.041823:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788574943.041825:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626840. 00040000:00000010:0.0:1788574943.041827:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788574943.041828:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4be00. 00000020:00000010:0.0:1788574943.041830:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788574943.041832:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e40. 10000000:01000000:0.0:1788574943.041834:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788574943.041837:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788574943.041838:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788574943.041839:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788574943.041841:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788574943.041853:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788574943.041855:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574943.041857:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788574943.041858:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574943.041860:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788574943.041862:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788574943.041863:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574943.041865:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788574944.577759:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788574944.577771:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788574944.577777:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574944.577780:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788574944.577782:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788574944.577783:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788574944.577787:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788574944.577789:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788574944.577791:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788574944.577795:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788574944.577797:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788574944.577811:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574944.577816:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788574944.577820:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574944.577823:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574944.577829:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574944.577836:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788574944.577844:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574944.577848:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788574944.577850:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788574944.577857:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692500. 00010000:00000001:0.0:1788574944.577866:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788574944.577869:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31c00. 00000020:00000001:0.0:1788574944.577874:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788574944.577876:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31c00 with handle 0x1441586093168cbf to hash 00000020:00000001:0.0:1788574944.577878:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788574944.577879:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574944.577882:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574944.577884:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574944.577902:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574944.577905:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574944.577914:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574944.577924:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788574944.577928:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788574944.577929:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788574944.577936:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574944.577937:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788574944.577939:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574944.577941:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788574944.577945:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574944.577949:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574944.577951:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788574944.577953:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574944.577963:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574944.577970:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788574944.577973:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574944.577977:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574944.577979:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574944.577983:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574944.577984:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574944.577986:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574944.577987:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574944.577992:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574944.577994:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788574944.577997:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574944.577999:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574944.578001:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574944.578004:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574944.578009:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810343168:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788574944.578013:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574944.578015:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574944.578017:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788574944.578019:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574944.578021:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574944.578025:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574944.578027:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788574944.578033:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788574944.578034:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788574944.578035:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788574944.578038:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574944.578040:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574944.578041:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=4 00000020:00000001:0.0:1788574944.578042:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574944.578044:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00010000:0.0:1788574944.578046:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578051:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788574944.578054:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578063:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788574944.578065:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578072:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574944.578074:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788574944.578076:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578080:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574944.578081:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788574944.578083:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788574944.578085:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788574944.578089:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31c00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788574944.578090:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574944.578092:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578095:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788574944.578096:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788574944.578097:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574944.578098:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31c00 refcount=5 00000020:00000001:0.0:1788574944.578099:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574944.578100:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688128 : -109714172863488 : ffff9c372db31c00) 00010000:00000001:0.0:1788574944.578102:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788574944.578103:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788574944.578107:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788574944.578111:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788574944.578115:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788574944.578117:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788574944.578118:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574944.578120:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788574944.578121:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788574944.578124:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788574944.578125:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788574944.578126:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788574944.578127:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788574944.578128:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31c00 with handle 0x1441586093168cbf from hash 00010000:00000001:0.0:1788574944.578130:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788574944.578131:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574944.578132:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574944.578133:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788574944.578134:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788574944.578136:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788574944.578137:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574944.578137:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574944.578138:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788574944.578139:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574944.578141:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788574944.578144:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574944.578144:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574944.578145:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788574944.578146:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788574944.578147:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788574944.578148:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574944.578148:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788574944.578150:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31c00/0x1441586093168cbf lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788574944.578154:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692500 count: 0 00010000:00000001:0.0:1788574944.578166:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788574944.578167:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788574944.578169:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574944.578171:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810343168/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574944.578176:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788574944.578178:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574944.578179:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788574944.578181:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574944.578183:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788574944.578187:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574944.578188:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574944.578190:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788574944.578194:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574944.578195:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788574944.578196:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788574944.578198:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788574944.578200:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788574945.473894:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574945.473899:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574945.473902:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574948.417754:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310087391, at 4310087424 00000100:00000010:0.0:1788574948.417769:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000800:00000200:0.0:1788574948.609950:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788574949.697709:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574949.697716:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574949.697718:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574949.697721:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574949.697724:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574949.697729:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15416 00000400:00000200:0.0:1788574949.697736:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574949.697740:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574949.697742:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574949.697758:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697761:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.697762:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697763:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574949.697766:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574949.697767:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574949.697769:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574949.697771:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574949.697773:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574949.697774:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574949.697776:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697777:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.697779:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697780:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574949.697782:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574949.697784:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697785:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574949.697799:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574949.697803:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574949.697805:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.697806:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574949.697808:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574949.697812:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574949.697815:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574949.697817:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574949.697818:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.697820:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574949.697822:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574949.697825:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574949.697830:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788574949.697834:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574949.697843:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810343936/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574949.697858:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574949.697862:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.697866:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810343936/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574949.697886:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574949.697888:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574949.697891:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574949.697892:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574949.697893:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574949.697894:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574949.697896:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15416 00000400:00000200:0.0:1788574949.697899:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574949.697901:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574949.697902:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574949.697904:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697905:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.697907:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697908:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574949.697909:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574949.697910:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574949.697911:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574949.697912:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574949.697913:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574949.697914:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574949.697916:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697917:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.697918:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697919:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574949.697924:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574949.697926:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574949.697927:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574949.697929:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574949.697933:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574949.697935:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.697936:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574949.697940:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574949.697942:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.697943:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574949.697946:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574949.697948:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574949.697950:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574949.697951:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.697952:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574949.697953:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574949.697955:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574949.697958:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788574949.697961:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574949.697964:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810344064/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574949.697969:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574949.697972:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.697980:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810344064/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574949.697995:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574949.698002:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574949.698006:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574949.698010:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574949.698013:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574949.698026:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574949.698037:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15416 00000400:00000200:0.0:1788574949.698046:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574949.698050:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574949.698064:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574949.698080:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698090:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698096:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698104:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574949.698118:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574949.698134:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574949.698145:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574949.698150:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574949.698155:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574949.698165:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574949.698174:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698176:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698179:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698186:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574949.698188:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574949.698190:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698191:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574949.698194:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574949.698197:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574949.698199:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698201:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574949.698208:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574949.698218:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698222:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574949.698225:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574949.698232:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574949.698240:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574949.698249:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698260:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574949.698264:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574949.698268:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574949.698280:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574949.698288:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574949.698296:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810344192/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574949.698307:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574949.698313:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.698319:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810344192/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574949.698330:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574949.698333:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574949.698338:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574949.698341:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574949.698345:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574949.698350:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574949.698356:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15416 00000400:00000200:0.0:1788574949.698392:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574949.698399:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574949.698402:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574949.698409:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698411:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698416:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698420:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574949.698427:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574949.698432:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574949.698435:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574949.698437:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574949.698447:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574949.698455:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574949.698463:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698469:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698476:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698482:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574949.698488:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574949.698492:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698498:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574949.698504:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574949.698509:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574949.698514:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698522:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574949.698528:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574949.698533:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698536:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574949.698541:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574949.698542:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574949.698544:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574949.698545:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698546:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574949.698549:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574949.698554:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574949.698564:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574949.698571:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574949.698577:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810344320/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574949.698676:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574949.698677:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.698681:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810344320/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574949.698692:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574949.698701:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574949.698704:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574949.698705:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574949.698706:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574949.698711:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574949.698717:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15416 00000400:00000200:0.0:1788574949.698731:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574949.698737:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574949.698742:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574949.698748:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698749:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698751:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698754:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574949.698760:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574949.698761:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574949.698762:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574949.698764:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574949.698765:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574949.698767:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574949.698777:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698779:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574949.698783:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698786:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574949.698800:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574949.698802:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574949.698804:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574949.698805:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574949.698807:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574949.698808:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698810:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574949.698814:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574949.698816:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698817:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574949.698819:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574949.698821:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574949.698822:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574949.698823:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574949.698824:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574949.698825:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574949.698826:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574949.698828:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574949.698831:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574949.698834:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810344448/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574949.698840:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574949.698842:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.698845:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810344448/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574949.698852:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.698856:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15426) 00000020:00000001:0.0:1788574950.593874:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574950.593879:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574950.593882:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574953.537718:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310092511, at 4310092544 00000100:00000010:0.0:1788574953.537734:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00080000:0.0:1788574954.817753:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574954.817763:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574954.817764:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574954.817767:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574954.817771:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574954.817776:0:189934:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15421 00000400:00000200:0.0:1788574954.817783:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574954.817801:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574954.817803:0:189934:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574954.817805:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817807:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.817809:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817810:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574954.817812:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574954.817814:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574954.817816:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574954.817818:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574954.817819:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574954.817821:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574954.817822:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817824:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.817825:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817826:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574954.817828:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574954.817830:0:189934:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817831:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574954.817833:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574954.817836:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574954.817837:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.817839:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574954.817840:0:189934:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574954.817845:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574954.817847:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574954.817850:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574954.817851:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.817853:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574954.817855:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574954.817858:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574954.817863:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574954.817871:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.817877:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810344576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574954.817884:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574954.817887:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.817889:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810344576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574954.817907:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574954.817909:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574954.817911:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574954.817912:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574954.817913:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574954.817914:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574954.817916:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15421 00000400:00000200:0.0:1788574954.817918:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574954.817920:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574954.817921:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574954.817922:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817924:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.817925:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817926:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574954.817927:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574954.817928:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574954.817929:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574954.817930:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574954.817931:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574954.817933:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574954.817934:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817936:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.817940:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817941:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574954.817943:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574954.817947:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574954.817950:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574954.817953:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574954.817960:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574954.817964:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.817968:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574954.817973:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574954.817979:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.817982:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574954.817989:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574954.817993:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574954.817997:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574954.818000:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818003:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574954.818005:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574954.818008:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574954.818022:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574954.818027:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.818033:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810344704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574954.818041:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574954.818043:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.818052:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810344704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574954.818064:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574954.818068:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574954.818075:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574954.818078:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574954.818084:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574954.818089:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574954.818093:0:189934:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15421 00000400:00000200:0.0:1788574954.818099:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574954.818104:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574954.818105:0:189934:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574954.818109:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818114:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818116:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818119:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574954.818123:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574954.818124:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574954.818126:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574954.818134:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574954.818140:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574954.818145:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574954.818151:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818153:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818155:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818159:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574954.818163:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574954.818169:0:189934:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818174:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574954.818178:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574954.818183:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574954.818189:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818194:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574954.818198:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574954.818203:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818207:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574954.818217:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574954.818221:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574954.818224:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574954.818227:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818230:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574954.818231:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574954.818234:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574954.818238:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574954.818243:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.818250:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810344832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574954.818266:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574954.818269:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.818275:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810344832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574954.818291:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574954.818296:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574954.818301:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574954.818305:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574954.818307:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574954.818312:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574954.818318:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15421 00000400:00000200:0.0:1788574954.818326:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574954.818332:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574954.818337:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574954.818342:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818345:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818349:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818352:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574954.818355:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574954.818362:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574954.818364:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574954.818367:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574954.818371:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574954.818374:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574954.818378:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818382:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818385:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818388:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574954.818392:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574954.818396:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818398:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574954.818401:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574954.818404:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574954.818407:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818410:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574954.818412:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574954.818415:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818418:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574954.818420:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574954.818424:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574954.818429:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574954.818430:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818433:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574954.818435:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574954.818437:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574954.818441:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574954.818447:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.818453:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810344960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574954.818468:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574954.818472:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.818482:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810344960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574954.818503:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574954.818507:0:189934:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574954.818511:0:189934:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574954.818513:0:189934:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574954.818517:0:189934:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574954.818522:0:189934:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574954.818527:0:189934:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15421 00000400:00000200:0.0:1788574954.818533:0:189934:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574954.818538:0:189934:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574954.818540:0:189934:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574954.818544:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818548:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818551:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818554:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574954.818558:0:189934:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574954.818561:0:189934:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574954.818564:0:189934:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574954.818567:0:189934:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574954.818570:0:189934:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574954.818572:0:189934:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574954.818574:0:189934:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818575:0:189934:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574954.818579:0:189934:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818583:0:189934:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574954.818586:0:189934:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574954.818590:0:189934:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574954.818594:0:189934:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574954.818602:0:189934:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574954.818606:0:189934:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574954.818610:0:189934:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818613:0:189934:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574954.818615:0:189934:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574954.818618:0:189934:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818621:0:189934:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574954.818624:0:189934:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574954.818628:0:189934:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574954.818633:0:189934:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574954.818662:0:189934:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574954.818667:0:189934:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574954.818670:0:189934:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574954.818674:0:189934:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574954.818681:0:189934:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574954.818686:0:189934:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.818697:0:189934:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810345088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574954.818709:0:189934:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574954.818714:0:189934:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.818720:0:189934:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810345088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574954.818737:0:189934:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.818741:0:189934:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15431) 00000020:00000001:0.0:1788574955.713836:0:248770:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574955.713839:0:248770:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574955.713841:0:248770:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574958.657743:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310097630, at 4310097664 00000100:00000010:0.0:1788574958.657760:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788574963.777758:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310102750, at 4310102784 00000100:00000010:0.0:1788574963.777773:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00080000:0.0:1788574966.658245:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574966.658256:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574966.658258:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574966.658261:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574966.658264:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574966.658268:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15436 00000400:00000200:0.0:1788574966.658276:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574966.658281:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574966.658284:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574966.658287:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658290:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658293:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658295:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574966.658298:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574966.658299:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574966.658301:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574966.658303:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574966.658306:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574966.658308:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574966.658311:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658313:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658315:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658317:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574966.658319:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574966.658322:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658325:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574966.658327:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574966.658329:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574966.658332:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658334:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574966.658336:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574966.658341:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574966.658349:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574966.658352:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574966.658353:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658355:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574966.658357:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574966.658359:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574966.658364:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574966.658367:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.658372:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810346624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574966.658380:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574966.658383:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.658388:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810346624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574966.658410:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574966.658413:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574966.658417:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574966.658418:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574966.658420:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574966.658422:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574966.658425:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15436 00000400:00000200:0.0:1788574966.658429:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574966.658432:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574966.658435:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574966.658439:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658442:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658444:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658446:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574966.658448:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574966.658449:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574966.658452:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574966.658460:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574966.658464:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574966.658469:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574966.658477:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658480:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658486:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658492:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574966.658497:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574966.658504:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658510:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574966.658513:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574966.658519:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574966.658536:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658540:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574966.658543:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574966.658549:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658554:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574966.658560:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574966.658564:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574966.658567:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574966.658571:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658576:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574966.658579:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574966.658582:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574966.658589:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574966.658595:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.658604:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810346752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574966.658632:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574966.658682:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.658690:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810346752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574966.658726:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574966.658732:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574966.658740:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574966.658744:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574966.658747:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574966.658751:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574966.658757:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15436 00000400:00000200:0.0:1788574966.658764:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574966.658768:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574966.658771:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574966.658775:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658779:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658785:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658789:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574966.658793:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574966.658794:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574966.658798:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574966.658801:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574966.658805:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574966.658809:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574966.658814:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658818:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.658824:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658828:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574966.658834:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574966.658838:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574966.658843:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574966.658852:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574966.658857:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574966.658862:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658866:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574966.658870:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574966.658876:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658880:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574966.658887:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574966.658891:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574966.658896:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574966.658899:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.658904:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574966.658908:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574966.658912:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574966.658919:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574966.658924:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.658933:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810346880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574966.658950:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574966.658955:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.658964:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810346880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574966.658978:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574966.658983:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574966.658988:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574966.658992:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574966.658995:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574966.659000:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574966.659006:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15436 00000400:00000200:0.0:1788574966.659015:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574966.659021:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574966.659025:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574966.659033:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659035:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.659040:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659044:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574966.659049:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574966.659052:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574966.659056:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574966.659060:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574966.659065:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574966.659069:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574966.659074:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659077:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.659082:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659087:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574966.659092:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574966.659098:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659100:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574966.659102:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574966.659106:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574966.659112:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659116:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574966.659122:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574966.659127:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659132:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574966.659140:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574966.659146:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574966.659152:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574966.659155:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659158:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574966.659161:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574966.659166:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574966.659173:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788574966.659184:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.659193:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810347008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574966.659209:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574966.659213:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.659219:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810347008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574966.659236:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574966.659240:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574966.659243:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574966.659245:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574966.659248:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574966.659252:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574966.659258:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15436 00000400:00000200:0.0:1788574966.659265:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574966.659271:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574966.659275:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574966.659280:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659286:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.659291:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659296:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574966.659301:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574966.659304:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574966.659309:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574966.659312:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574966.659317:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574966.659323:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574966.659331:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659337:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574966.659349:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659354:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574966.659360:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574966.659368:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574966.659374:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574966.659379:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574966.659383:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574966.659386:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659389:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574966.659391:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574966.659395:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659398:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574966.659403:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574966.659406:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574966.659411:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574966.659414:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.659417:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574966.659420:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574966.659422:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574966.659427:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574966.659433:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.659441:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810347136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574966.659462:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574966.659467:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.659477:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810347136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574966.659498:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574966.659505:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15443) 00000100:00000040:0.0:1788574970.497775:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310109471, at 4310109504 00000100:00000010:0.0:1788574970.497794:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00080000:0.0:1788574971.713804:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.713833:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574971.713836:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.713840:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.713845:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.713853:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15438 00000400:00000200:0.0:1788574971.713863:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574971.713869:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.713873:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.713878:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.713883:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.713885:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.713888:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574971.713892:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.713896:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574971.713899:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.713903:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574971.713906:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574971.713910:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574971.713914:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.713917:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.713920:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.713922:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574971.713925:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574971.713929:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574971.713931:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.713935:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.713938:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.713941:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.713943:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574971.713945:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.713952:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574971.713963:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574971.713967:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.713968:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.713979:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.713982:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.713989:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.713998:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574971.714005:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.714016:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810347264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.714026:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.714031:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.714039:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810347264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.714070:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.714073:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.714078:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574971.714080:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.714082:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.714085:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.714087:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15438 00000400:00000200:0.0:1788574971.714095:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574971.714102:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.714105:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.714110:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714113:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.714121:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714128:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574971.714134:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.714137:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574971.714141:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.714150:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574971.714160:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574971.714169:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574971.714185:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714194:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.714204:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714213:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574971.714219:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574971.714229:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714239:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.714242:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.714251:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.714257:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714261:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.714266:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574971.714273:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714282:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.714290:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574971.714302:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574971.714308:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.714317:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714326:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.714328:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.714330:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.714337:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574971.714343:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.714354:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810347392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.714368:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.714373:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.714382:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810347392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.714431:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.714439:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.714452:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574971.714458:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.714462:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.714467:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.714473:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15438 00000400:00000200:0.0:1788574971.714497:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574971.714508:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.714521:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.714526:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714531:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.714541:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714545:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574971.714552:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.714557:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574971.714561:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.714569:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574971.714579:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574971.714591:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574971.714604:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714617:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.714622:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714628:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574971.714633:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574971.714680:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574971.714689:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.714697:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.714702:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.714713:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714721:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.714730:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574971.714734:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714739:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.714756:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574971.714765:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574971.714774:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.714778:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.714780:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.714791:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.714799:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.714807:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574971.714815:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.714834:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810347520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.714844:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.714846:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.714852:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810347520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.714871:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.714877:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.714889:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574971.714901:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.714913:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.714922:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.714932:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15438 00000400:00000200:0.0:1788574971.714948:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574971.714963:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.714972:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.714994:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715004:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.715016:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715026:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574971.715041:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.715044:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574971.715052:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.715056:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574971.715060:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574971.715063:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574971.715069:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715072:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.715081:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715084:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574971.715092:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574971.715105:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715112:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.715119:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.715128:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.715142:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715149:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.715162:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574971.715178:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715187:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.715193:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574971.715197:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574971.715199:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.715200:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715201:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.715202:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.715204:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.715207:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788574971.715222:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.715245:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810347648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.715267:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.715268:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.715271:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810347648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.715278:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.715281:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.715288:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574971.715289:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.715298:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.715307:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.715315:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15438 00000400:00000200:0.0:1788574971.715331:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574971.715336:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.715339:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.715344:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715350:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.715353:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715361:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574971.715365:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.715372:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574971.715374:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.715379:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574971.715388:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574971.715391:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574971.715398:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715405:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.715411:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715415:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574971.715422:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574971.715432:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574971.715441:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.715445:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.715451:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.715460:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715463:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.715464:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574971.715466:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715467:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.715470:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574971.715475:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574971.715478:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.715483:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.715485:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.715491:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.715494:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.715501:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788574971.715505:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.715512:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810347776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.715573:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.715576:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.715583:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810347776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.715607:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.715613:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15448) 00000100:00000040:0.0:1788574975.553710:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310114527, at 4310114560 00000100:00000010:0.0:1788574975.553722:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00000040:0.0:1788574980.673758:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310119646, at 4310119680 00000100:00000010:0.0:1788574980.673786:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00000040:0.0:1788574985.793735:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310124766, at 4310124800 00000100:00000010:0.0:1788574985.793756:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000800:00000200:0.0:1788574986.497926:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788574987.073757:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574987.073764:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574987.073768:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574987.073773:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574987.073780:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574987.073789:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15453 00000400:00000200:0.0:1788574987.073800:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574987.073808:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574987.073813:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574987.073817:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.073820:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.073823:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.073829:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574987.073833:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574987.073837:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574987.073840:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574987.073847:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574987.073851:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574987.073854:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574987.073862:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.073869:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.073873:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.073877:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574987.073882:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574987.073889:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574987.073893:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574987.073905:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574987.073919:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574987.073926:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.073930:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574987.073936:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574987.073946:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788574987.073954:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574987.073959:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574987.073961:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.073965:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574987.073969:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574987.073973:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574987.073980:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574987.073988:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.073997:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810349184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574987.074008:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574987.074012:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.074017:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810349184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574987.074039:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574987.074043:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574987.074046:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574987.074048:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574987.074050:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574987.074053:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574987.074055:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15453 00000400:00000200:0.0:1788574987.074062:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574987.074065:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574987.074077:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574987.074086:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074092:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074106:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074113:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574987.074120:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574987.074124:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574987.074130:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574987.074136:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574987.074142:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574987.074148:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574987.074155:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074162:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074169:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074175:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574987.074181:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574987.074185:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074188:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574987.074192:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574987.074199:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574987.074206:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074211:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574987.074217:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574987.074224:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074229:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574987.074237:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788574987.074244:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574987.074248:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574987.074250:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074255:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574987.074259:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574987.074264:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574987.074272:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574987.074280:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.074292:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810349312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574987.074322:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574987.074328:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.074347:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810349312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574987.074371:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574987.074377:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574987.074382:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574987.074386:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574987.074390:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574987.074396:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574987.074403:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15453 00000400:00000200:0.0:1788574987.074423:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574987.074433:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574987.074439:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574987.074446:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074453:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074460:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074466:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574987.074471:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574987.074473:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574987.074477:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574987.074481:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574987.074488:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574987.074494:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574987.074504:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074511:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074518:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074523:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574987.074537:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574987.074548:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074554:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574987.074560:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574987.074567:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574987.074574:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074579:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574987.074585:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574987.074594:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074599:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574987.074606:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788574987.074613:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574987.074620:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574987.074625:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074632:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574987.074668:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574987.074673:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574987.074684:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574987.074692:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.074705:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810349440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574987.074737:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574987.074740:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.074751:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810349440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574987.074770:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574987.074776:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574987.074785:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574987.074791:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574987.074796:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574987.074801:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574987.074813:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15453 00000400:00000200:0.0:1788574987.074821:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574987.074831:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574987.074837:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574987.074846:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074852:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074861:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074869:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574987.074876:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574987.074880:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574987.074886:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574987.074891:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574987.074898:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574987.074905:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574987.074922:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074929:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.074936:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074944:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574987.074951:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574987.074960:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574987.074968:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574987.074974:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574987.074981:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574987.074990:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.074997:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574987.075002:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574987.075008:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.075012:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574987.075019:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788574987.075024:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574987.075032:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574987.075043:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.075047:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574987.075051:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574987.075056:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574987.075065:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574987.075073:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.075088:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810349568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574987.075109:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574987.075115:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.075126:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810349568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574987.075152:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574987.075161:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574987.075169:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574987.075173:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574987.075178:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574987.075186:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574987.075193:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15453 00000400:00000200:0.0:1788574987.075205:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574987.075214:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574987.075221:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574987.075228:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.075235:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.075242:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.075250:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574987.075257:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574987.075261:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574987.075267:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574987.075272:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574987.075281:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574987.075294:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574987.075305:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.075312:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574987.075320:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574987.075327:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574987.075335:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574987.075344:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574987.075352:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574987.075357:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574987.075364:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574987.075371:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.075377:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574987.075383:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574987.075391:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.075395:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574987.075404:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788574987.075430:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574987.075438:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574987.075444:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574987.075450:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574987.075455:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574987.075461:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574987.075471:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788574987.075479:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.075492:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810349696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574987.075515:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574987.075521:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.075532:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810349696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574987.075553:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574987.075567:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15463) 00000100:00000040:0.0:1788574992.897736:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310131871, at 4310131904 00000100:00000010:0.0:1788574992.897753:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00080000:0.0:1788574994.241804:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574994.241812:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574994.241814:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574994.241817:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574994.241821:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574994.241827:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15460 00000400:00000200:0.0:1788574994.241834:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574994.241840:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574994.241843:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574994.241845:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.241848:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.241850:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.241852:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574994.241855:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574994.241857:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574994.241860:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574994.241864:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574994.241866:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574994.241869:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574994.241872:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.241874:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.241876:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.241878:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574994.241881:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574994.241885:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574994.241888:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574994.241891:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574994.241913:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574994.241916:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.241918:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574994.241920:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574994.241926:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788574994.241929:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574994.241931:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574994.241933:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.241935:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574994.241937:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574994.241941:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574994.241948:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788574994.241953:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.241960:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810350592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574994.241968:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574994.241971:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.241975:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810350592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574994.241994:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574994.241997:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574994.241999:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574994.242001:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574994.242002:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574994.242003:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574994.242005:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15460 00000400:00000200:0.0:1788574994.242008:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574994.242010:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574994.242012:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574994.242014:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242016:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242023:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242025:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574994.242027:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574994.242029:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574994.242034:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574994.242036:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574994.242040:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574994.242043:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574994.242049:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242052:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242056:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242061:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574994.242066:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574994.242069:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242073:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574994.242075:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574994.242081:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574994.242088:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242091:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574994.242097:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574994.242100:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242101:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574994.242104:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788574994.242106:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574994.242110:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574994.242114:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242115:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574994.242118:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574994.242122:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574994.242127:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788574994.242141:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.242152:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810350720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574994.242166:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574994.242170:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.242175:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810350720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574994.242186:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574994.242190:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574994.242194:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574994.242197:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574994.242199:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574994.242203:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574994.242207:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15460 00000400:00000200:0.0:1788574994.242214:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574994.242218:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574994.242222:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574994.242226:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242229:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242234:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242238:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574994.242240:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574994.242241:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574994.242242:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574994.242244:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574994.242246:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574994.242250:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574994.242255:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242256:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242257:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242258:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574994.242265:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574994.242270:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242274:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574994.242277:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574994.242280:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574994.242284:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242288:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574994.242292:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574994.242296:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242301:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574994.242306:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788574994.242310:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574994.242312:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574994.242316:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242319:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574994.242321:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574994.242325:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574994.242329:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788574994.242334:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.242340:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810350848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574994.242353:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574994.242357:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.242364:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810350848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574994.242377:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574994.242380:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574994.242385:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574994.242387:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574994.242388:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574994.242390:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574994.242400:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15460 00000400:00000200:0.0:1788574994.242405:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574994.242410:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574994.242413:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574994.242419:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242423:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242427:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242431:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574994.242434:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574994.242436:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574994.242439:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574994.242442:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574994.242446:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574994.242450:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574994.242456:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242460:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242465:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242469:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574994.242471:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574994.242474:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242478:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574994.242480:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574994.242482:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574994.242485:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242489:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574994.242491:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574994.242497:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242498:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574994.242502:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788574994.242506:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574994.242508:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574994.242512:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242515:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574994.242516:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574994.242518:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574994.242522:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788574994.242526:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.242533:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810350976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574994.242548:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574994.242552:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.242560:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810350976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574994.242572:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574994.242575:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574994.242579:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574994.242582:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574994.242585:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574994.242589:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574994.242593:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15460 00000400:00000200:0.0:1788574994.242598:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574994.242603:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574994.242607:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574994.242612:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242615:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242619:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242622:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574994.242625:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574994.242627:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574994.242630:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574994.242633:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574994.242673:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574994.242684:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574994.242692:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242697:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574994.242701:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242704:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574994.242707:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574994.242714:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574994.242719:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574994.242738:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574994.242742:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574994.242748:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242753:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574994.242756:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574994.242759:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242762:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574994.242768:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574994.242771:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574994.242776:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574994.242778:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574994.242779:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574994.242780:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574994.242782:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574994.242785:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788574994.242788:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.242793:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810351104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574994.242799:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574994.242802:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.242814:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810351104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574994.242834:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574994.242845:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15470) 00000100:00000040:0.0:1788574998.081770:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310137054, at 4310137088 00000100:00000010:0.0:1788574998.081786:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00080000:0.0:1788574999.361758:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574999.361770:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574999.361773:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574999.361777:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574999.361782:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574999.361789:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15465 00000400:00000200:0.0:1788574999.361799:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574999.361804:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574999.361807:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574999.361809:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.361812:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.361814:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.361816:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574999.361819:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574999.361820:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788574999.361823:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574999.361825:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788574999.361827:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788574999.361828:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788574999.361830:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.361832:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.361833:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.361835:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574999.361837:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788574999.361839:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574999.361841:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574999.361843:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574999.361860:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574999.361862:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.361864:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574999.361866:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574999.361871:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788574999.361875:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788574999.361879:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574999.361881:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.361884:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574999.361887:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574999.361891:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574999.361899:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788574999.361906:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.361928:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810351232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574999.361937:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574999.361940:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.361956:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810351232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574999.361975:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574999.361977:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574999.361980:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574999.361983:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574999.361985:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574999.361986:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574999.361988:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15465 00000400:00000200:0.0:1788574999.361992:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574999.361994:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574999.361996:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574999.361998:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362000:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362007:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362009:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574999.362011:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574999.362012:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788574999.362014:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574999.362015:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788574999.362017:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788574999.362019:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788574999.362024:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362029:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362033:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362034:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574999.362038:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788574999.362042:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362047:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574999.362051:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574999.362058:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574999.362064:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362067:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574999.362072:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574999.362078:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362082:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574999.362092:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788574999.362098:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574999.362103:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574999.362106:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362108:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574999.362110:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574999.362113:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574999.362120:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788574999.362125:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.362132:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810351360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574999.362158:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574999.362165:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.362175:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810351360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574999.362190:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574999.362195:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574999.362200:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574999.362202:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574999.362205:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574999.362209:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574999.362214:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15465 00000400:00000200:0.0:1788574999.362244:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574999.362251:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574999.362256:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574999.362263:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362266:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362271:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362273:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574999.362275:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574999.362276:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788574999.362277:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574999.362278:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788574999.362280:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788574999.362281:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788574999.362283:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362284:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362286:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362292:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574999.362301:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788574999.362306:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362309:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574999.362311:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574999.362314:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574999.362318:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362321:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574999.362325:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574999.362329:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362333:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574999.362340:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788574999.362344:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788574999.362348:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574999.362350:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362353:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574999.362354:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574999.362357:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574999.362364:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788574999.362370:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.362379:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810351488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574999.362401:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574999.362405:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.362411:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810351488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574999.362433:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574999.362449:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574999.362457:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574999.362462:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574999.362467:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574999.362474:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574999.362488:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15465 00000400:00000200:0.0:1788574999.362495:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574999.362503:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574999.362508:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574999.362516:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362519:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362524:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362527:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574999.362531:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574999.362534:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788574999.362538:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574999.362541:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788574999.362546:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788574999.362549:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788574999.362556:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362559:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362565:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362569:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574999.362576:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788574999.362585:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362593:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574999.362600:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574999.362605:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574999.362612:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362617:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574999.362622:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574999.362667:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362670:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574999.362675:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788574999.362680:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788574999.362687:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574999.362699:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362703:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574999.362707:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574999.362712:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574999.362719:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788574999.362727:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.362738:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810351616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574999.362753:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574999.362759:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.362767:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810351616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574999.362784:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574999.362789:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574999.362796:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574999.362799:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574999.362803:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574999.362806:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574999.362812:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15465 00000400:00000200:0.0:1788574999.362822:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788574999.362829:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574999.362835:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574999.362839:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362845:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362848:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362854:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788574999.362859:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574999.362863:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788574999.362867:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574999.362870:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788574999.362875:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788574999.362887:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788574999.362896:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362900:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574999.362904:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362910:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788574999.362915:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788574999.362922:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788574999.362926:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574999.362930:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574999.362936:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574999.362941:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362958:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574999.362963:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574999.362969:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.362974:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574999.362983:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788574999.362989:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788574999.362996:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574999.362999:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.363005:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574999.363009:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574999.363013:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574999.363021:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788574999.363027:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.363038:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810351744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574999.363055:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574999.363061:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.363071:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810351744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574999.363096:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574999.363109:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15476) 00000100:00000040:0.0:1788575003.201742:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310142174, at 4310142208 00000100:00000010:0.0:1788575003.201763:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00000040:0.0:1788575008.321740:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310147295, at 4310147328 00000100:00000010:0.0:1788575008.321762:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00000040:0.0:1788575014.337748:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310153311, at 4310153344 00000100:00000010:0.0:1788575014.337765:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00080000:0.0:1788575015.745777:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575015.745790:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575015.745793:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575015.745797:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575015.745801:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575015.745809:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15482 00000400:00000200:0.0:1788575015.745818:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575015.745824:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575015.745827:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575015.745831:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.745835:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.745839:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.745842:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575015.745846:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575015.745848:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575015.745851:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575015.745855:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575015.745858:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575015.745860:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575015.745864:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.745865:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.745868:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.745870:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575015.745891:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575015.745896:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575015.745899:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575015.745902:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575015.745906:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575015.745909:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.745912:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575015.745916:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575015.745924:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575015.745928:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575015.745934:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575015.745935:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.745938:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575015.745942:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575015.745947:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575015.745956:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575015.745964:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.745978:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810353920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575015.745990:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575015.745994:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.746001:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810353920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575015.746034:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575015.746038:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575015.746042:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575015.746044:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575015.746045:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575015.746048:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575015.746050:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15482 00000400:00000200:0.0:1788575015.746055:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575015.746066:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575015.746069:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575015.746073:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746075:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746079:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746085:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575015.746088:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575015.746092:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575015.746096:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575015.746099:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575015.746106:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575015.746111:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575015.746118:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746123:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746127:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746133:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575015.746139:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575015.746146:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746148:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575015.746150:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575015.746158:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575015.746163:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746168:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575015.746174:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575015.746181:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746185:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575015.746190:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575015.746196:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575015.746201:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575015.746205:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746208:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575015.746216:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575015.746220:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575015.746227:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575015.746236:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.746246:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810354048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575015.746268:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575015.746273:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.746281:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810354048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575015.746297:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575015.746301:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575015.746305:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575015.746309:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575015.746312:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575015.746317:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575015.746323:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15482 00000400:00000200:0.0:1788575015.746332:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575015.746336:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575015.746338:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575015.746342:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746344:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746350:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746354:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575015.746360:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575015.746364:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575015.746370:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575015.746376:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575015.746382:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575015.746389:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575015.746412:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746423:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746430:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746436:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575015.746442:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575015.746450:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746455:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575015.746459:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575015.746464:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575015.746468:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746482:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575015.746486:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575015.746490:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746492:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575015.746497:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575015.746502:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575015.746505:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575015.746509:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746511:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575015.746512:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575015.746514:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575015.746521:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575015.746528:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.746539:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810354176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575015.746559:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575015.746570:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.746579:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810354176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575015.746604:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575015.746606:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575015.746615:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575015.746621:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575015.746624:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575015.746629:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575015.746633:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15482 00000400:00000200:0.0:1788575015.746673:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575015.746676:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575015.746678:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575015.746681:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746685:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746687:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746693:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575015.746696:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575015.746698:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575015.746703:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575015.746706:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575015.746710:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575015.746712:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575015.746718:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746723:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746728:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746730:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575015.746734:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575015.746741:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746747:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575015.746752:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575015.746756:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575015.746760:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746762:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575015.746766:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575015.746776:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746780:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575015.746784:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575015.746788:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575015.746794:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575015.746797:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.746800:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575015.746804:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575015.746807:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575015.746813:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575015.746819:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.746828:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810354304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575015.746853:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575015.746857:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.746864:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810354304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575015.746884:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575015.746888:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575015.746891:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575015.746894:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575015.746899:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575015.746905:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575015.746907:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15482 00000400:00000200:0.0:1788575015.746913:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575015.746919:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575015.746922:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575015.746927:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746931:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746936:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746942:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575015.746944:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575015.746945:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575015.746946:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575015.746948:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575015.746949:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575015.746951:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575015.746959:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746961:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575015.746964:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746968:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575015.746973:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575015.746979:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575015.746984:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575015.746988:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575015.746991:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575015.746995:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.747001:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575015.747004:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575015.747007:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.747012:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575015.747017:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575015.747021:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575015.747027:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575015.747029:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575015.747032:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575015.747036:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575015.747041:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575015.747047:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575015.747053:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.747060:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810354432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575015.747086:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575015.747089:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.747095:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810354432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575015.747122:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.747128:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15492) 00000100:00000040:0.0:1788575019.585771:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310158558, at 4310158592 00000100:00000010:0.0:1788575019.585790:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788575020.865738:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575020.865746:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575020.865748:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575020.865751:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575020.865755:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575020.865761:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15487 00000400:00000200:0.0:1788575020.865768:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575020.865773:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575020.865776:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575020.865779:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865782:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.865783:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865785:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575020.865788:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575020.865790:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575020.865792:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575020.865794:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575020.865796:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575020.865798:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575020.865800:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865815:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.865816:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865818:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575020.865820:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575020.865823:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865825:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575020.865827:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575020.865830:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575020.865832:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.865833:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575020.865835:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575020.865840:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575020.865843:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575020.865846:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575020.865848:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.865850:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575020.865853:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575020.865856:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575020.865862:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575020.865868:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.865875:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810354560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575020.865883:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575020.865886:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.865890:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810354560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575020.865911:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575020.865914:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575020.865917:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575020.865918:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575020.865919:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575020.865921:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575020.865926:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15487 00000400:00000200:0.0:1788575020.865929:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575020.865932:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575020.865933:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575020.865935:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865937:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.865938:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865940:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575020.865941:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575020.865943:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575020.865944:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575020.865946:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575020.865947:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575020.865949:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575020.865951:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865952:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.865954:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865956:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575020.865958:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575020.865960:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575020.865962:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575020.865966:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575020.865971:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575020.865975:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.865977:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575020.865981:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575020.865987:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.865991:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575020.865995:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575020.865999:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575020.866003:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575020.866010:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866014:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575020.866016:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575020.866020:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575020.866025:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575020.866031:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.866040:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810354688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575020.866052:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575020.866056:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.866062:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810354688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575020.866071:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575020.866074:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575020.866080:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575020.866085:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575020.866090:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575020.866096:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575020.866103:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15487 00000400:00000200:0.0:1788575020.866114:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575020.866122:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575020.866128:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575020.866135:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866140:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866145:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866149:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575020.866152:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575020.866155:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575020.866158:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575020.866161:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575020.866166:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575020.866177:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575020.866196:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866203:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866207:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866211:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575020.866213:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575020.866220:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866224:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575020.866230:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575020.866233:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575020.866240:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866243:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575020.866246:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575020.866252:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866258:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575020.866261:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575020.866268:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575020.866271:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575020.866274:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866277:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575020.866281:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575020.866282:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575020.866286:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575020.866291:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.866299:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810354816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575020.866313:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575020.866317:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.866328:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810354816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575020.866359:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575020.866367:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575020.866370:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575020.866371:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575020.866373:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575020.866377:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575020.866382:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15487 00000400:00000200:0.0:1788575020.866390:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575020.866397:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575020.866399:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575020.866405:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866407:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866408:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866411:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575020.866414:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575020.866416:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575020.866417:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575020.866420:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575020.866422:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575020.866426:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575020.866430:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866434:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866437:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866439:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575020.866444:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575020.866447:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866453:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575020.866456:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575020.866460:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575020.866465:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866469:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575020.866476:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575020.866480:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866482:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575020.866487:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575020.866493:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575020.866497:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575020.866500:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866503:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575020.866506:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575020.866509:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575020.866512:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575020.866515:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.866524:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810354944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575020.866533:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575020.866538:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.866546:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810354944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575020.866561:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575020.866565:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575020.866574:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575020.866577:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575020.866579:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575020.866583:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575020.866603:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15487 00000400:00000200:0.0:1788575020.866612:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575020.866618:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575020.866620:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575020.866627:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866633:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866655:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866661:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575020.866664:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575020.866665:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575020.866666:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575020.866670:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575020.866676:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575020.866679:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575020.866686:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866689:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575020.866690:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866691:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575020.866693:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575020.866696:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575020.866701:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575020.866706:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575020.866711:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575020.866715:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866719:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575020.866722:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575020.866726:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866730:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575020.866734:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575020.866740:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575020.866744:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575020.866747:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575020.866752:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575020.866754:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575020.866758:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575020.866763:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575020.866768:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.866775:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810355072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575020.866795:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575020.866799:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.866806:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810355072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575020.866819:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.866823:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15497) 00000100:00000040:0.0:1788575024.705773:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310163678, at 4310163712 00000100:00000010:0.0:1788575024.705795:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000800:00000200:0.0:1788575025.409923:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788575025.985762:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575025.985773:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575025.985776:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575025.985780:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575025.985784:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575025.985792:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15492 00000400:00000200:0.0:1788575025.985801:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575025.985807:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575025.985810:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575025.985813:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985815:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.985817:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985819:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575025.985822:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575025.985823:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575025.985826:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575025.985829:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575025.985830:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575025.985832:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575025.985834:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985854:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.985857:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985859:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575025.985863:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575025.985866:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985869:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575025.985872:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575025.985876:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575025.985880:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.985882:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575025.985885:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575025.985892:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575025.985896:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575025.985899:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575025.985900:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.985903:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575025.985906:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575025.985910:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575025.985915:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575025.985921:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.985929:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810355200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575025.985937:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575025.985940:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.985943:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810355200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575025.985962:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575025.985965:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575025.985967:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575025.985969:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575025.985970:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575025.985972:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575025.985978:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15492 00000400:00000200:0.0:1788575025.985981:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575025.985984:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575025.985986:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575025.985989:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985991:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.985994:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.985996:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575025.985999:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575025.986004:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575025.986007:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575025.986012:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575025.986027:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575025.986034:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575025.986040:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986045:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986050:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986054:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575025.986060:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575025.986065:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986069:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575025.986073:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575025.986080:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575025.986084:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986087:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575025.986091:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575025.986096:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986099:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575025.986104:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575025.986107:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575025.986111:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575025.986119:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986124:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575025.986127:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575025.986129:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575025.986132:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575025.986136:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.986147:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810355328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575025.986161:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575025.986165:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.986172:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810355328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575025.986195:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575025.986200:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575025.986208:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575025.986213:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575025.986218:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575025.986226:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575025.986234:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15492 00000400:00000200:0.0:1788575025.986253:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575025.986261:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575025.986267:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575025.986275:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986282:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986289:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986296:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575025.986300:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575025.986304:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575025.986309:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575025.986314:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575025.986318:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575025.986331:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575025.986342:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986348:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986355:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986360:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575025.986366:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575025.986372:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986379:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575025.986383:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575025.986387:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575025.986391:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986392:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575025.986395:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575025.986399:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986400:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575025.986404:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575025.986408:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575025.986413:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575025.986416:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986418:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575025.986421:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575025.986424:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575025.986433:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575025.986442:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.986452:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810355456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575025.986472:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575025.986477:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.986486:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810355456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575025.986500:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575025.986512:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575025.986517:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575025.986521:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575025.986525:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575025.986531:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575025.986536:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15492 00000400:00000200:0.0:1788575025.986543:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575025.986548:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575025.986554:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575025.986560:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986566:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986572:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986575:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575025.986576:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575025.986577:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575025.986578:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575025.986580:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575025.986581:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575025.986583:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575025.986585:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986589:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986593:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986597:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575025.986601:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575025.986606:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986611:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575025.986613:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575025.986618:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575025.986623:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986626:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575025.986632:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575025.986659:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986662:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575025.986667:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575025.986670:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575025.986673:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575025.986677:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986682:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575025.986687:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575025.986691:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575025.986700:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575025.986720:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.986730:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810355584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575025.986740:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575025.986745:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.986751:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810355584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575025.986768:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575025.986774:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575025.986783:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575025.986786:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575025.986789:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575025.986796:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575025.986804:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15492 00000400:00000200:0.0:1788575025.986811:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575025.986816:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575025.986821:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575025.986829:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986834:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986837:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986849:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575025.986856:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575025.986861:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575025.986866:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575025.986871:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575025.986877:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575025.986885:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575025.986894:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986898:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575025.986903:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986906:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575025.986911:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575025.986917:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575025.986920:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575025.986923:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575025.986927:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575025.986933:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986935:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575025.986938:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575025.986941:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986944:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575025.986948:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575025.986953:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575025.986958:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575025.986961:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575025.986965:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575025.986969:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575025.986973:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575025.986979:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575025.986986:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.986994:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810355712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575025.987024:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575025.987029:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.987036:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810355712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575025.987051:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575025.987057:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15502) 00000100:00000040:0.0:1788575029.825731:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310168798, at 4310168832 00000100:00000010:0.0:1788575029.825749:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00080000:0.0:1788575031.105750:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575031.105758:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575031.105760:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575031.105763:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575031.105766:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575031.105771:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15497 00000400:00000200:0.0:1788575031.105779:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575031.105785:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575031.105789:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575031.105792:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105796:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.105798:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105801:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575031.105805:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575031.105808:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575031.105811:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575031.105814:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575031.105817:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575031.105820:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575031.105834:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105851:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.105853:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105854:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575031.105856:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575031.105858:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105860:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575031.105862:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575031.105865:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575031.105867:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.105868:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575031.105870:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575031.105874:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575031.105877:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575031.105880:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575031.105881:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.105883:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575031.105886:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575031.105889:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575031.105894:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575031.105900:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.105906:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810355840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575031.105913:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575031.105916:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.105919:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810355840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575031.105934:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575031.105937:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575031.105939:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575031.105940:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575031.105941:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575031.105943:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575031.105948:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15497 00000400:00000200:0.0:1788575031.105951:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575031.105953:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575031.105954:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575031.105956:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105957:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.105959:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105960:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575031.105962:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575031.105962:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575031.105964:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575031.105965:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575031.105967:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575031.105968:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575031.105970:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105971:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.105973:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105974:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575031.105976:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575031.105978:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575031.105980:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575031.105981:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575031.105987:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575031.105991:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.105993:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575031.105998:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575031.106001:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106002:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575031.106005:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575031.106008:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575031.106012:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575031.106016:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106019:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575031.106021:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575031.106025:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575031.106032:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575031.106035:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.106041:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810355968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575031.106052:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575031.106055:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.106060:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810355968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575031.106068:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575031.106071:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575031.106075:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575031.106079:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575031.106080:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575031.106081:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575031.106083:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15497 00000400:00000200:0.0:1788575031.106087:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575031.106091:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575031.106094:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575031.106095:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106097:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106100:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106103:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575031.106105:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575031.106106:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575031.106108:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575031.106110:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575031.106114:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575031.106120:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575031.106125:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106127:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106128:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106130:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575031.106134:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575031.106137:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106141:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575031.106143:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575031.106147:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575031.106149:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106151:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575031.106153:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575031.106157:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106159:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575031.106164:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575031.106167:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575031.106170:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575031.106172:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106174:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575031.106177:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575031.106180:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575031.106184:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575031.106188:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.106196:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810356096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575031.106205:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575031.106207:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.106210:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810356096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575031.106219:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575031.106225:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575031.106230:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575031.106233:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575031.106235:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575031.106239:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575031.106243:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15497 00000400:00000200:0.0:1788575031.106272:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575031.106277:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575031.106280:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575031.106282:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106287:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106288:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106289:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575031.106293:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575031.106295:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575031.106298:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575031.106301:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575031.106303:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575031.106305:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575031.106310:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106314:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106317:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106320:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575031.106322:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575031.106327:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106329:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575031.106332:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575031.106335:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575031.106337:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106341:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575031.106346:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575031.106349:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106353:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575031.106357:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575031.106360:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575031.106361:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575031.106365:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106367:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575031.106369:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575031.106371:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575031.106375:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575031.106380:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.106387:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810356224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575031.106402:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575031.106404:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.106414:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810356224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575031.106439:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575031.106441:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575031.106443:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575031.106443:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575031.106444:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575031.106445:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575031.106449:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15497 00000400:00000200:0.0:1788575031.106454:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575031.106459:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575031.106461:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575031.106465:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106469:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106475:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106484:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575031.106489:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575031.106490:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575031.106494:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575031.106497:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575031.106500:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575031.106504:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575031.106510:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106513:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575031.106517:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106519:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575031.106523:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575031.106527:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575031.106528:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575031.106531:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575031.106533:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575031.106537:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106539:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575031.106541:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575031.106542:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106543:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575031.106546:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575031.106550:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575031.106554:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575031.106557:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575031.106559:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575031.106560:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575031.106565:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575031.106570:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575031.106575:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.106589:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810356352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575031.106626:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575031.106630:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.106652:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810356352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575031.106658:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.106661:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15507) 00000100:00080000:0.0:1788575032.194326:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575032.194332:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575032.194335:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575032.194338:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575032.194342:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575032.194346:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15502 00000400:00000200:0.0:1788575032.194354:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575032.194360:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575032.194364:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575032.194367:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194370:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194372:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194375:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575032.194378:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575032.194380:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575032.194381:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575032.194384:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575032.194386:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575032.194388:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575032.194391:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194393:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194395:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194397:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575032.194406:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575032.194410:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194413:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575032.194416:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575032.194418:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575032.194421:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194423:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575032.194425:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575032.194429:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575032.194432:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575032.194434:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575032.194437:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194439:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575032.194440:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575032.194443:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575032.194447:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575032.194451:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.194457:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810356608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575032.194465:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575032.194468:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.194472:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810356608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575032.194491:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575032.194494:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575032.194498:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575032.194499:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575032.194501:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575032.194504:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575032.194506:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15502 00000400:00000200:0.0:1788575032.194511:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575032.194521:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575032.194526:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575032.194540:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194545:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194552:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194558:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575032.194563:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575032.194566:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575032.194570:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575032.194574:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575032.194578:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575032.194583:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575032.194588:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194594:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194599:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194607:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575032.194613:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575032.194619:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194625:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575032.194629:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575032.194689:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575032.194695:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194700:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575032.194707:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575032.194714:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194720:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575032.194726:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575032.194732:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575032.194737:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575032.194739:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194744:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575032.194752:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575032.194757:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575032.194765:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575032.194773:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.194783:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810356736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575032.194802:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575032.194806:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.194817:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810356736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575032.194840:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575032.194856:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575032.194861:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575032.194865:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575032.194867:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575032.194872:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575032.194878:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15502 00000400:00000200:0.0:1788575032.194885:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575032.194898:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575032.194902:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575032.194908:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194913:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194919:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194923:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575032.194926:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575032.194927:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575032.194929:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575032.194931:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575032.194935:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575032.194940:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575032.194952:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194956:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.194962:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194967:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575032.194972:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575032.194978:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575032.194982:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575032.194985:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575032.194989:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575032.194992:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.194995:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575032.194998:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575032.195000:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195001:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575032.195003:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575032.195006:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575032.195010:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575032.195012:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195016:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575032.195018:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575032.195020:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575032.195025:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575032.195030:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.195039:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810356864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575032.195048:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575032.195050:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.195057:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810356864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575032.195069:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575032.195072:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575032.195082:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575032.195086:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575032.195088:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575032.195093:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575032.195096:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15502 00000400:00000200:0.0:1788575032.195102:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575032.195107:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575032.195110:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575032.195116:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195120:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.195124:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195128:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575032.195133:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575032.195137:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575032.195141:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575032.195143:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575032.195145:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575032.195148:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575032.195154:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195159:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.195166:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195171:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575032.195175:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575032.195182:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195184:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575032.195186:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575032.195190:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575032.195195:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195197:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575032.195201:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575032.195209:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195211:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575032.195214:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575032.195226:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575032.195234:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575032.195238:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195243:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575032.195247:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575032.195251:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575032.195258:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575032.195263:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.195272:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810356992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575032.195287:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575032.195291:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.195299:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810356992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575032.195316:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575032.195322:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575032.195328:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575032.195332:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575032.195336:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575032.195341:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575032.195347:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15502 00000400:00000200:0.0:1788575032.195355:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575032.195362:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575032.195366:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575032.195373:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195377:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.195382:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195395:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575032.195399:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575032.195402:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575032.195408:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575032.195412:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575032.195417:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575032.195420:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575032.195425:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195429:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575032.195434:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195437:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575032.195440:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575032.195445:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575032.195449:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575032.195451:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575032.195455:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575032.195459:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195462:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575032.195467:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575032.195471:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195474:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575032.195478:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575032.195482:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575032.195486:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575032.195489:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575032.195491:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575032.195492:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575032.195495:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575032.195501:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575032.195506:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.195512:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810357120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575032.195536:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575032.195540:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.195545:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810357120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575032.195556:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575032.195561:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15508) 00000100:00000040:0.0:1788575036.033759:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310175007, at 4310175040 00000100:00000010:0.0:1788575036.033776:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00080000:0.0:1788575037.249742:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575037.249755:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575037.249758:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575037.249762:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575037.249768:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575037.249776:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15503 00000400:00000200:0.0:1788575037.249787:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575037.249794:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575037.249798:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575037.249801:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.249804:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.249807:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.249809:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575037.249814:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575037.249816:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575037.249824:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575037.249828:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575037.249831:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575037.249834:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575037.249837:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.249857:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.249859:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.249861:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575037.249864:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575037.249867:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575037.249870:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575037.249873:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575037.249877:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575037.249880:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.249883:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575037.249885:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575037.249892:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575037.249895:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575037.249900:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575037.249901:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.249905:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575037.249908:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575037.249913:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575037.249922:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575037.249930:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.249940:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810357248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575037.249950:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575037.249955:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.249960:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810357248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575037.250001:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575037.250005:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575037.250008:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575037.250009:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575037.250011:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575037.250013:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575037.250021:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15503 00000400:00000200:0.0:1788575037.250025:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575037.250029:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575037.250031:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575037.250035:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250038:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250040:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250042:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575037.250045:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575037.250047:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575037.250049:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575037.250052:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575037.250056:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575037.250060:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575037.250065:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250070:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250074:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250078:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575037.250082:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575037.250087:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250090:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575037.250093:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575037.250101:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575037.250105:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250108:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575037.250114:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575037.250118:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250123:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575037.250130:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575037.250133:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575037.250138:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575037.250145:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250148:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575037.250152:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575037.250154:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575037.250161:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575037.250168:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.250177:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810357376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575037.250190:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575037.250194:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.250201:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810357376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575037.250214:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575037.250219:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575037.250224:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575037.250227:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575037.250229:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575037.250234:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575037.250240:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15503 00000400:00000200:0.0:1788575037.250267:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575037.250273:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575037.250277:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575037.250284:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250289:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250295:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250302:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575037.250307:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575037.250311:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575037.250315:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575037.250318:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575037.250321:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575037.250330:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575037.250339:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250342:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250348:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250353:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575037.250358:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575037.250373:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250377:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575037.250380:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575037.250385:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575037.250392:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250395:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575037.250397:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575037.250400:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250403:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575037.250409:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575037.250414:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575037.250419:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575037.250421:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250425:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575037.250427:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575037.250429:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575037.250436:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575037.250443:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.250454:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810357504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575037.250468:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575037.250473:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.250482:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810357504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575037.250500:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575037.250511:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575037.250519:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575037.250523:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575037.250527:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575037.250532:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575037.250539:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15503 00000400:00000200:0.0:1788575037.250548:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575037.250556:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575037.250562:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575037.250568:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250574:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250578:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250582:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575037.250587:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575037.250591:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575037.250593:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575037.250595:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575037.250598:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575037.250603:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575037.250612:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250616:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250623:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250629:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575037.250662:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575037.250669:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250674:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575037.250676:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575037.250680:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575037.250682:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250685:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575037.250695:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575037.250700:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250703:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575037.250708:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575037.250712:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575037.250714:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575037.250717:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250718:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575037.250720:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575037.250723:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575037.250727:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575037.250733:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.250739:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810357632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575037.250748:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575037.250750:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.250753:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810357632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575037.250764:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575037.250768:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575037.250772:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575037.250775:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575037.250776:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575037.250782:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575037.250786:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15503 00000400:00000200:0.0:1788575037.250790:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575037.250795:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575037.250798:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575037.250803:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250807:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250811:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250819:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575037.250822:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575037.250824:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575037.250827:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575037.250829:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575037.250833:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575037.250837:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575037.250842:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250844:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575037.250847:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250850:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575037.250853:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575037.250856:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575037.250860:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575037.250861:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575037.250864:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575037.250867:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250871:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575037.250873:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575037.250877:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250880:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575037.250885:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575037.250888:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575037.250891:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575037.250893:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575037.250897:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575037.250900:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575037.250902:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575037.250907:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575037.250911:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.250919:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810357760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575037.250941:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575037.250944:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.250949:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810357760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575037.250960:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575037.250978:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15513) 00000100:00000040:0.0:1788575041.089760:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310180062, at 4310180096 00000100:00000010:0.0:1788575041.089782:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000100:00000040:0.0:1788575046.209831:0:267854:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310185182, at 4310185216 00000100:00000010:0.0:1788575046.209836:0:267854:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00000040:0.0:1788575051.329763:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310190302, at 4310190336 00000100:00000010:0.0:1788575051.329783:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00000040:0.0:1788575058.177734:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310197151, at 4310197184 00000100:00000010:0.0:1788575058.177752:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000800:00000200:0.0:1788575063.297934:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575063.617738:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310202590, at 4310202624 00000100:00000010:0.0:1788575063.617754:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788575064.897868:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575064.897881:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575064.897885:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575064.897891:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575064.897898:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575064.897910:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15531 00000400:00000200:0.0:1788575064.897923:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575064.897932:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575064.897937:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575064.897941:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.897946:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.897959:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.897962:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575064.897967:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575064.897971:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575064.897974:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575064.897988:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575064.897991:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575064.897995:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575064.898001:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898004:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898008:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898012:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575064.898016:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575064.898021:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898025:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575064.898030:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575064.898035:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575064.898039:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898043:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575064.898046:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575064.898054:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575064.898059:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575064.898065:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575064.898068:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898072:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575064.898076:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575064.898082:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575064.898092:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575064.898101:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.898114:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810361216/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575064.898126:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575064.898139:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.898145:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810361216/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575064.898170:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575064.898175:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575064.898179:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575064.898181:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575064.898183:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575064.898187:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575064.898190:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15531 00000400:00000200:0.0:1788575064.898196:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575064.898201:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575064.898207:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575064.898213:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898217:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898222:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898227:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575064.898231:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575064.898235:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575064.898240:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575064.898245:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575064.898251:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575064.898256:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575064.898264:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898271:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898278:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898283:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575064.898287:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575064.898292:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898307:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575064.898312:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575064.898322:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575064.898328:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898333:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575064.898340:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575064.898346:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898351:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575064.898356:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575064.898363:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575064.898370:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575064.898373:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898379:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575064.898382:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575064.898386:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575064.898395:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575064.898405:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.898415:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810361344/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575064.898431:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575064.898436:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.898446:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810361344/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575064.898467:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575064.898474:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575064.898482:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575064.898488:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575064.898492:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575064.898501:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575064.898510:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15531 00000400:00000200:0.0:1788575064.898521:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575064.898540:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575064.898545:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575064.898555:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898558:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898564:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898569:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575064.898574:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575064.898580:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575064.898586:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575064.898591:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575064.898611:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575064.898619:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575064.898626:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898632:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898678:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898683:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575064.898689:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575064.898694:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898700:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575064.898707:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575064.898714:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575064.898721:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898726:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575064.898728:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575064.898734:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898740:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575064.898748:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575064.898755:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575064.898763:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575064.898768:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.898774:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575064.898785:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575064.898790:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575064.898800:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575064.898808:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.898819:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810361472/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575064.898840:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575064.898843:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.898851:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810361472/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575064.898876:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575064.898879:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575064.898885:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575064.898889:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575064.898893:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575064.898900:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575064.898906:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15531 00000400:00000200:0.0:1788575064.898915:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575064.898923:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575064.898928:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575064.898935:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898940:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.898946:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.898951:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575064.898957:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575064.898962:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575064.898966:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575064.898972:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575064.898978:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575064.898984:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575064.899000:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899005:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.899012:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899017:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575064.899022:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575064.899030:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899034:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575064.899038:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575064.899044:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575064.899049:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899053:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575064.899058:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575064.899063:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899066:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575064.899073:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575064.899078:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575064.899084:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575064.899087:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899091:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575064.899094:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575064.899099:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575064.899107:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575064.899113:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.899122:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810361600/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575064.899137:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575064.899141:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.899149:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810361600/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575064.899170:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575064.899175:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575064.899185:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575064.899187:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575064.899189:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575064.899196:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575064.899202:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15531 00000400:00000200:0.0:1788575064.899209:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575064.899216:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575064.899221:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575064.899227:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899231:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.899237:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899243:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575064.899249:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575064.899253:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575064.899257:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575064.899262:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575064.899266:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575064.899272:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575064.899282:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899287:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575064.899293:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899297:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575064.899303:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575064.899309:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575064.899315:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575064.899319:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575064.899323:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575064.899329:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899335:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575064.899338:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575064.899351:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899356:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575064.899364:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575064.899370:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575064.899377:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575064.899381:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.899385:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575064.899389:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575064.899393:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575064.899401:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575064.899408:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.899418:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810361728/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575064.899432:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575064.899434:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.899440:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810361728/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575064.899459:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575064.899467:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15541) 00000100:00000040:0.0:1788575068.737751:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310207710, at 4310207744 00000100:00000010:0.0:1788575068.737774:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00080000:0.0:1788575070.017771:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575070.017781:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575070.017783:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575070.017786:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575070.017791:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575070.017798:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15536 00000400:00000200:0.0:1788575070.017807:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575070.017812:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575070.017815:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575070.017819:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.017839:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.017842:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.017844:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575070.017848:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575070.017849:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575070.017852:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575070.017855:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575070.017857:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575070.017858:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575070.017861:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.017862:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.017863:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.017865:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575070.017867:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575070.017869:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575070.017871:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575070.017873:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575070.017876:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575070.017879:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.017882:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575070.017884:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575070.017890:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575070.017894:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575070.017898:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575070.017900:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.017904:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575070.017907:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575070.017912:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575070.017920:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575070.017927:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.017936:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810361856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575070.017953:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575070.017956:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.017960:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810361856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575070.017981:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575070.017986:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575070.017989:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575070.017991:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575070.017992:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575070.017995:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575070.017998:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15536 00000400:00000200:0.0:1788575070.018003:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575070.018006:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575070.018010:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575070.018013:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018017:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018020:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018022:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575070.018025:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575070.018026:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575070.018027:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575070.018030:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575070.018031:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575070.018033:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575070.018035:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018036:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018038:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018039:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575070.018053:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575070.018059:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018063:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575070.018067:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575070.018072:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575070.018076:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018078:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575070.018082:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575070.018087:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018098:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575070.018108:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575070.018119:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575070.018141:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575070.018147:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018158:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575070.018162:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575070.018169:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575070.018185:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575070.018194:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.018208:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810361984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575070.018221:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575070.018229:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.018244:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810361984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575070.018271:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575070.018277:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575070.018284:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575070.018289:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575070.018296:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575070.018302:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575070.018325:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15536 00000400:00000200:0.0:1788575070.018350:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575070.018364:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575070.018368:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575070.018373:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018383:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018392:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018401:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575070.018406:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575070.018408:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575070.018414:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575070.018416:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575070.018422:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575070.018427:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575070.018431:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018433:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018435:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018442:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575070.018449:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575070.018459:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018465:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575070.018470:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575070.018478:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575070.018486:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018491:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575070.018496:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575070.018502:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018506:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575070.018514:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575070.018523:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575070.018530:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575070.018547:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.018553:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575070.018555:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575070.018559:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575070.018570:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575070.018579:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.018592:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810362112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575070.018617:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575070.018624:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.018634:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810362112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575070.018777:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575070.018788:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575070.018810:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575070.018817:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575070.018823:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575070.018831:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575070.018840:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15536 00000400:00000200:0.0:1788575070.018851:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575070.018869:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575070.018877:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575070.018886:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018894:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018897:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018899:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575070.018910:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575070.018914:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575070.018918:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575070.018926:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575070.018934:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575070.018952:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575070.018970:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.018978:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.018988:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019000:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575070.019009:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575070.019024:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019037:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575070.019045:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575070.019053:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575070.019062:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019070:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575070.019077:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575070.019085:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019090:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575070.019105:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575070.019120:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575070.019130:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575070.019136:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019143:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575070.019147:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575070.019154:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575070.019168:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575070.019178:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.019206:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810362240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575070.019245:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575070.019251:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.019270:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810362240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575070.019291:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575070.019308:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575070.019316:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575070.019321:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575070.019328:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575070.019339:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575070.019348:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15536 00000400:00000200:0.0:1788575070.019364:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575070.019379:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575070.019389:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575070.019402:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019413:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.019423:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019430:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575070.019438:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575070.019444:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575070.019451:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575070.019457:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575070.019465:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575070.019474:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575070.019482:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019491:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575070.019499:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019506:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575070.019516:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575070.019522:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575070.019527:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575070.019533:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575070.019545:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575070.019551:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019554:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575070.019564:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575070.019569:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019575:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575070.019583:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575070.019591:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575070.019603:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575070.019609:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.019617:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575070.019620:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575070.019626:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575070.019670:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575070.019678:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.019691:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810362368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575070.019708:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575070.019714:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.019740:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810362368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575070.019765:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575070.019774:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15546) 00000100:00000040:0.0:1788575073.857772:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310212831, at 4310212864 00000100:00000010:0.0:1788575073.857813:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788575075.137847:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575075.137863:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575075.137865:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575075.137869:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575075.137874:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575075.137881:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15541 00000400:00000200:0.0:1788575075.137892:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575075.137898:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575075.137902:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575075.137937:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.137941:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.137944:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.137946:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575075.137952:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575075.137955:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575075.137958:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575075.137962:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575075.137964:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575075.137966:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575075.137968:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.137970:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.137971:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.137973:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575075.137976:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575075.137982:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575075.137984:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575075.137988:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575075.137991:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575075.137993:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.137995:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575075.137998:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575075.138007:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575075.138010:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575075.138014:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575075.138015:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138019:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575075.138023:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575075.138028:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575075.138039:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575075.138048:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.138060:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810362496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575075.138081:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575075.138086:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138093:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810362496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575075.138118:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575075.138121:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575075.138124:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575075.138125:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575075.138127:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575075.138129:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575075.138131:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15541 00000400:00000200:0.0:1788575075.138135:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575075.138138:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575075.138140:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575075.138143:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138144:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138146:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138148:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575075.138150:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575075.138151:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575075.138152:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575075.138154:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575075.138156:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575075.138158:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575075.138160:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138162:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138164:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138169:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575075.138177:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575075.138182:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138185:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575075.138187:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575075.138194:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575075.138198:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138201:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575075.138207:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575075.138211:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138215:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575075.138222:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575075.138224:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575075.138229:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575075.138233:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138238:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575075.138242:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575075.138245:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575075.138251:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575075.138256:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.138264:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810362624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575075.138274:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575075.138276:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138280:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810362624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575075.138294:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575075.138297:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575075.138301:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575075.138303:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575075.138305:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575075.138307:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575075.138317:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15541 00000400:00000200:0.0:1788575075.138323:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575075.138328:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575075.138333:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575075.138340:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138342:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138346:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138350:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575075.138354:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575075.138357:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575075.138360:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575075.138363:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575075.138367:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575075.138370:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575075.138375:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138378:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138381:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138384:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575075.138386:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575075.138392:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138397:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575075.138399:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575075.138404:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575075.138411:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138416:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575075.138420:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575075.138427:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138433:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575075.138441:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575075.138448:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575075.138452:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575075.138459:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138461:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575075.138462:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575075.138465:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575075.138471:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575075.138476:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.138483:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810362752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575075.138493:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575075.138495:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138501:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810362752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575075.138511:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575075.138516:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575075.138530:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575075.138533:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575075.138536:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575075.138540:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575075.138544:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15541 00000400:00000200:0.0:1788575075.138550:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575075.138556:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575075.138558:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575075.138561:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138564:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138570:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138575:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575075.138579:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575075.138582:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575075.138586:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575075.138589:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575075.138592:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575075.138604:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575075.138610:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138614:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138619:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138624:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575075.138630:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575075.138657:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138659:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575075.138661:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575075.138663:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575075.138665:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138667:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575075.138669:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575075.138671:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138673:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575075.138676:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575075.138678:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575075.138680:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575075.138681:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138682:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575075.138684:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575075.138686:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575075.138689:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575075.138692:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.138697:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810362880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575075.138703:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575075.138705:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138711:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810362880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575075.138721:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575075.138730:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575075.138737:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575075.138740:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575075.138743:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575075.138747:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575075.138752:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15541 00000400:00000200:0.0:1788575075.138756:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575075.138758:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575075.138760:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575075.138764:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138766:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138767:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138769:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575075.138771:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575075.138773:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575075.138775:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575075.138778:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575075.138780:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575075.138782:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575075.138788:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138792:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575075.138795:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138797:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575075.138800:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575075.138804:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575075.138806:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575075.138807:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575075.138824:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575075.138829:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138831:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575075.138837:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575075.138842:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138845:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575075.138848:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575075.138850:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575075.138852:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575075.138853:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.138854:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575075.138855:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575075.138857:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575075.138860:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575075.138865:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.138871:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810363008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575075.138882:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575075.138884:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138891:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810363008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575075.138901:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575075.138906:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15551) 00000020:00000001:0.0:1788575076.545863:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575076.545867:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575076.545873:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575080.385764:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310219359, at 4310219392 00000100:00000010:0.0:1788575080.385780:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788575081.665905:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575081.665910:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575081.665914:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575081.793788:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575081.793799:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575081.793801:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575081.793805:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575081.793831:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575081.793839:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15548 00000400:00000200:0.0:1788575081.793848:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575081.793854:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575081.793857:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575081.793860:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.793863:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.793865:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.793868:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575081.793874:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575081.793876:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575081.793880:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575081.793884:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575081.793887:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575081.793889:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575081.793893:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.793897:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.793899:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.793901:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575081.793905:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575081.793909:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575081.793912:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575081.793936:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575081.793943:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575081.793946:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.793948:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575081.793950:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575081.793956:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575081.793959:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575081.793963:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575081.793965:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.793972:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575081.793975:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575081.793979:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575081.793988:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575081.793995:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.794005:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810363904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575081.794015:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575081.794019:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.794022:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810363904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575081.794041:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575081.794043:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575081.794046:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575081.794047:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575081.794049:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575081.794051:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575081.794053:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15548 00000400:00000200:0.0:1788575081.794056:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575081.794059:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575081.794061:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575081.794063:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794065:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794067:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794069:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575081.794071:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575081.794072:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575081.794073:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575081.794075:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575081.794076:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575081.794078:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575081.794088:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794090:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794092:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794094:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575081.794096:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575081.794098:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794099:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575081.794100:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575081.794108:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575081.794111:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794115:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575081.794122:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575081.794129:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794134:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575081.794137:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575081.794145:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575081.794148:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575081.794152:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794158:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575081.794163:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575081.794168:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575081.794176:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575081.794185:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.794194:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810364032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575081.794227:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575081.794233:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.794243:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810364032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575081.794264:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575081.794266:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575081.794282:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575081.794283:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575081.794290:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575081.794304:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575081.794311:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15548 00000400:00000200:0.0:1788575081.794316:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575081.794326:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575081.794330:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575081.794337:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794346:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794353:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794360:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575081.794367:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575081.794371:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575081.794377:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575081.794383:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575081.794395:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575081.794405:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575081.794408:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794413:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794419:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794424:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575081.794433:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575081.794444:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794449:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575081.794455:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575081.794463:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575081.794472:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794486:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575081.794494:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575081.794511:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794517:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575081.794526:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575081.794532:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575081.794546:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575081.794551:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794555:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575081.794560:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575081.794567:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575081.794582:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575081.794590:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.794599:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810364160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575081.794625:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575081.794628:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.794671:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810364160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575081.794681:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575081.794684:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575081.794689:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575081.794691:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575081.794693:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575081.794700:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575081.794709:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15548 00000400:00000200:0.0:1788575081.794726:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575081.794737:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575081.794748:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575081.794755:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794757:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794759:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794773:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575081.794781:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575081.794787:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575081.794789:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575081.794792:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575081.794808:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575081.794817:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575081.794825:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794832:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.794834:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794835:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575081.794838:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575081.794844:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575081.794846:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575081.794847:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575081.794851:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575081.794854:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794855:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575081.794858:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575081.794862:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794867:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575081.794879:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575081.794889:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575081.794901:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575081.794907:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.794915:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575081.794918:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575081.794940:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575081.794949:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575081.794960:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.794980:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810364288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575081.795024:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575081.795030:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.795039:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810364288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575081.795071:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575081.795078:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575081.795086:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575081.795089:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575081.795091:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575081.795102:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575081.795111:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15548 00000400:00000200:0.0:1788575081.795126:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575081.795135:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575081.795138:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575081.795146:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.795159:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.795167:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.795172:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575081.795175:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575081.795179:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575081.795185:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575081.795189:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575081.795196:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575081.795202:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575081.795218:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.795231:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575081.795241:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575081.795245:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575081.795264:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575081.795273:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575081.795281:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575081.795285:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575081.795288:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575081.795290:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.795292:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575081.795294:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575081.795301:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.795304:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575081.795306:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575081.795311:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575081.795315:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575081.795318:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.795325:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575081.795328:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575081.795330:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575081.795341:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575081.795346:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.795358:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810364416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575081.795365:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575081.795369:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.795374:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810364416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575081.795390:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575081.795402:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15558) 00000100:00000040:0.0:1788575085.633723:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310224607, at 4310224640 00000100:00000010:0.0:1788575085.633735:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788575086.785798:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575086.785801:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575086.785803:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575090.753682:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310229726, at 4310229760 00000100:00000010:0.0:1788575090.753701:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788575091.905804:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575091.905807:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575091.905809:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575092.033756:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575092.033764:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575092.033766:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575092.033769:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575092.033772:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575092.033783:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15558 00000400:00000200:0.0:1788575092.033791:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575092.033795:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575092.033798:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575092.033800:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033802:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.033803:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033805:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575092.033808:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575092.033809:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575092.033811:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575092.033813:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575092.033815:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575092.033816:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575092.033818:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033819:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.033820:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033822:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575092.033824:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575092.033826:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033843:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575092.033846:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575092.033848:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575092.033850:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.033851:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575092.033853:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575092.033857:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575092.033860:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575092.033863:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575092.033865:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.033867:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575092.033869:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575092.033872:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575092.033877:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575092.033882:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.033888:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810365184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575092.033894:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575092.033897:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.033900:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810365184/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575092.033917:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575092.033919:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575092.033921:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575092.033922:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575092.033923:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575092.033924:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575092.033926:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15558 00000400:00000200:0.0:1788575092.033933:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575092.033935:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575092.033937:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575092.033938:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033944:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.033946:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033948:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575092.033949:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575092.033950:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575092.033951:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575092.033953:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575092.033957:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575092.033959:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575092.033964:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033966:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.033970:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033971:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575092.033972:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575092.033974:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575092.033976:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575092.033979:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575092.033984:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575092.033988:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.033992:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575092.033998:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575092.034005:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034010:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575092.034015:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575092.034022:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575092.034026:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575092.034030:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034036:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575092.034040:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575092.034044:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575092.034051:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575092.034055:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.034068:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810365312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575092.034082:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575092.034086:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.034105:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810365312/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575092.034112:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575092.034114:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575092.034116:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575092.034119:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575092.034121:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575092.034126:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575092.034135:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15558 00000400:00000200:0.0:1788575092.034178:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575092.034185:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575092.034188:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575092.034192:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034193:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034197:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034199:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575092.034205:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575092.034206:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575092.034207:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575092.034210:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575092.034213:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575092.034216:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575092.034221:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034226:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034228:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034237:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575092.034242:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575092.034245:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034248:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575092.034250:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575092.034254:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575092.034257:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034259:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575092.034263:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575092.034267:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034269:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575092.034272:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575092.034275:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575092.034286:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575092.034289:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034295:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575092.034297:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575092.034302:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575092.034308:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575092.034317:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.034325:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810365440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575092.034334:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575092.034337:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.034342:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810365440/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575092.034353:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575092.034355:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575092.034359:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575092.034360:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575092.034362:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575092.034372:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575092.034376:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15558 00000400:00000200:0.0:1788575092.034381:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575092.034385:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575092.034389:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575092.034391:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034395:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034397:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034402:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575092.034406:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575092.034408:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575092.034410:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575092.034414:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575092.034416:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575092.034418:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575092.034423:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034426:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034430:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034433:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575092.034437:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575092.034441:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034446:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575092.034450:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575092.034454:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575092.034461:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034466:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575092.034470:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575092.034477:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034482:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575092.034489:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575092.034497:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575092.034502:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575092.034505:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034508:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575092.034510:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575092.034513:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575092.034519:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575092.034523:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.034527:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810365568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575092.034532:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575092.034536:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.034542:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810365568/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575092.034569:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575092.034577:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575092.034583:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575092.034587:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575092.034592:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575092.034600:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575092.034607:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15558 00000400:00000200:0.0:1788575092.034616:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575092.034623:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575092.034627:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575092.034631:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034670:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034675:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034679:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575092.034683:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575092.034684:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575092.034691:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575092.034695:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575092.034699:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575092.034702:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575092.034706:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034710:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575092.034713:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034716:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575092.034721:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575092.034726:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575092.034730:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575092.034733:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575092.034736:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575092.034739:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034742:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575092.034746:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575092.034747:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034749:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575092.034755:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575092.034758:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575092.034762:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575092.034763:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.034767:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575092.034768:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575092.034772:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575092.034777:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575092.034781:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.034787:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810365696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575092.034794:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575092.034798:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.034804:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810365696/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575092.034819:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.034823:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15568) 00000100:00000040:0.0:1788575095.873796:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310234846, at 4310234880 00000100:00000010:0.0:1788575095.873818:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788575097.025774:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575097.025777:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575097.025780:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575098.498353:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575098.498360:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575098.498363:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575098.498367:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575098.498371:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575098.498376:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15568 00000400:00000200:0.0:1788575098.498385:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575098.498392:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575098.498395:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575098.498399:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498411:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498414:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498419:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575098.498422:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575098.498425:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575098.498427:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575098.498429:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575098.498431:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575098.498434:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575098.498437:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498439:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498441:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498449:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575098.498453:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575098.498457:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498460:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575098.498463:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575098.498466:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575098.498469:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498472:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575098.498474:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575098.498480:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575098.498482:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575098.498485:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575098.498487:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498489:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575098.498491:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575098.498494:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575098.498499:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575098.498503:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.498510:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810366592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575098.498519:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575098.498521:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.498530:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810366592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575098.498551:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575098.498555:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575098.498558:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575098.498559:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575098.498561:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575098.498564:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575098.498568:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15568 00000400:00000200:0.0:1788575098.498578:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575098.498584:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575098.498589:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575098.498595:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498598:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498602:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498606:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575098.498611:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575098.498613:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575098.498617:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575098.498622:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575098.498625:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575098.498629:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575098.498668:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498673:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498678:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498681:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575098.498685:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575098.498689:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498694:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575098.498698:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575098.498704:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575098.498708:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498711:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575098.498717:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575098.498721:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498725:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575098.498730:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575098.498733:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575098.498738:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575098.498745:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498749:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575098.498751:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575098.498753:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575098.498758:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575098.498764:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.498772:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810366720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575098.498784:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575098.498788:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.498794:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810366720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575098.498810:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575098.498813:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575098.498819:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575098.498821:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575098.498823:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575098.498827:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575098.498832:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15568 00000400:00000200:0.0:1788575098.498839:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575098.498846:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575098.498850:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575098.498857:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498862:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498867:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498872:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575098.498876:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575098.498878:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575098.498882:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575098.498884:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575098.498886:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575098.498894:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575098.498898:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498902:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.498907:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498910:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575098.498915:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575098.498920:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575098.498926:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575098.498928:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575098.498931:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575098.498936:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498939:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575098.498943:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575098.498948:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498952:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575098.498958:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575098.498964:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575098.498971:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575098.498973:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.498978:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575098.498979:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575098.498983:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575098.498990:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575098.498996:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.499003:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810366848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575098.499014:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575098.499019:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.499026:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810366848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575098.499040:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575098.499051:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575098.499060:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575098.499063:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575098.499065:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575098.499069:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575098.499074:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15568 00000400:00000200:0.0:1788575098.499082:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575098.499087:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575098.499092:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575098.499096:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499099:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.499104:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499108:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575098.499114:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575098.499117:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575098.499120:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575098.499123:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575098.499125:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575098.499129:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575098.499134:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499138:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.499140:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499142:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575098.499146:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575098.499151:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499155:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575098.499160:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575098.499165:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575098.499170:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499175:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575098.499184:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575098.499190:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499195:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575098.499215:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575098.499220:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575098.499226:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575098.499229:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499232:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575098.499236:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575098.499240:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575098.499247:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575098.499255:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.499263:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810366976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575098.499275:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575098.499280:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.499289:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810366976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575098.499307:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575098.499311:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575098.499316:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575098.499318:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575098.499322:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575098.499329:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575098.499336:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15568 00000400:00000200:0.0:1788575098.499343:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575098.499350:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575098.499356:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575098.499364:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499369:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.499374:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499384:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575098.499386:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575098.499390:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575098.499394:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575098.499398:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575098.499403:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575098.499407:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575098.499413:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499415:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575098.499417:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499421:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575098.499427:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575098.499433:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575098.499438:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575098.499442:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575098.499447:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575098.499452:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499456:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575098.499461:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575098.499465:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499469:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575098.499476:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575098.499481:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575098.499486:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575098.499489:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575098.499495:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575098.499499:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575098.499503:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575098.499511:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575098.499518:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.499526:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810367104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575098.499565:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575098.499570:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.499581:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810367104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575098.499606:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575098.499615:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15575) 00000020:00000001:0.0:1788575102.145833:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575102.145836:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575102.145839:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575102.337745:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310241311, at 4310241344 00000100:00000010:0.0:1788575102.337761:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215de0. 00000800:00000200:0.0:1788575103.233938:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575107.265828:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575107.265831:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575107.265834:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575107.649699:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310246622, at 4310246656 00000100:00000010:0.0:1788575107.649712:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215720. 00000020:00000001:0.0:1788575112.385777:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575112.385781:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575112.385785:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575112.769768:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310251742, at 4310251776 00000100:00000010:0.0:1788575112.769799:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322156c0. 00000100:00080000:0.0:1788575114.049801:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.049815:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575114.049818:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.049822:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.049826:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.049832:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15580 00000400:00000200:0.0:1788575114.049841:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575114.049846:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.049870:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.049874:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.049878:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.049880:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.049883:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575114.049887:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.049889:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575114.049893:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.049897:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575114.049900:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575114.049902:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575114.049905:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.049907:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.049908:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.049910:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575114.049912:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575114.049915:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575114.049916:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.049919:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.049922:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.049924:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.049925:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575114.049927:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.049934:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575114.049938:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575114.049943:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.049945:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.049949:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.049952:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.049964:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.049972:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575114.049979:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.049992:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810368512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.050000:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.050004:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050007:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810368512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.050024:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.050026:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.050029:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575114.050030:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.050031:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.050032:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.050034:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15580 00000400:00000200:0.0:1788575114.050037:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575114.050039:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.050041:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.050043:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050045:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050047:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050049:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575114.050052:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.050053:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575114.050055:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.050058:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575114.050062:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575114.050065:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575114.050081:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050086:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050092:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050104:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575114.050116:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575114.050124:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050129:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.050135:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.050144:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.050147:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050149:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.050153:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575114.050156:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050158:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.050162:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575114.050166:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575114.050171:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.050175:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050180:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.050190:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.050193:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.050201:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575114.050206:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.050214:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810368640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.050226:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.050230:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050240:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810368640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.050260:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.050264:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.050271:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575114.050272:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.050274:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.050284:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.050289:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15580 00000400:00000200:0.0:1788575114.050296:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575114.050309:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.050312:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.050315:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050318:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050321:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050323:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575114.050327:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.050329:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575114.050333:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.050335:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575114.050340:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575114.050343:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575114.050347:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050350:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050354:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050356:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575114.050358:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575114.050360:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050365:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.050366:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.050371:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.050373:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050375:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.050376:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575114.050380:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050384:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.050389:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575114.050399:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575114.050402:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.050405:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050409:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.050412:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.050415:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.050422:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575114.050429:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.050438:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810368768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.050447:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.050453:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050459:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810368768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.050497:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.050502:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.050505:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575114.050508:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.050512:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.050518:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.050524:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15580 00000400:00000200:0.0:1788575114.050540:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575114.050548:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.050553:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.050555:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050559:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050562:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050563:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575114.050564:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.050565:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575114.050571:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.050574:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575114.050577:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575114.050581:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575114.050584:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050588:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050591:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050601:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575114.050605:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575114.050609:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050611:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.050612:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.050614:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.050619:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050623:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.050626:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575114.050631:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050632:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.050666:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575114.050668:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575114.050670:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.050672:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050677:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.050679:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.050683:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.050686:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575114.050692:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.050699:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810368896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.050708:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.050712:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050718:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810368896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.050734:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.050739:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.050743:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575114.050744:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.050746:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.050747:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.050752:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15580 00000400:00000200:0.0:1788575114.050758:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575114.050763:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.050766:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.050770:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050774:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050780:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050786:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575114.050791:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.050794:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575114.050798:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.050802:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575114.050806:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575114.050811:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575114.050815:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050817:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.050821:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050824:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575114.050828:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575114.050834:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575114.050838:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.050846:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.050850:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.050853:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050854:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.050856:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575114.050858:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050860:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.050865:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575114.050871:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575114.050877:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.050879:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.050884:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.050887:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.050890:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.050896:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575114.050903:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.050912:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810369024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.050921:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.050925:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050932:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810369024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.050946:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.050954:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15590) 00000020:00000001:0.0:1788575117.505820:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575117.505824:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575117.505828:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575117.889740:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310256862, at 4310256896 00000100:00000010:0.0:1788575117.889749:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215900. 00000100:00080000:0.0:1788575119.169749:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575119.169758:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575119.169760:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575119.169776:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575119.169780:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575119.169787:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15585 00000400:00000200:0.0:1788575119.169795:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575119.169800:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575119.169803:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575119.169806:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169808:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.169811:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169813:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575119.169817:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575119.169818:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575119.169821:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575119.169824:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575119.169826:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575119.169828:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575119.169830:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169832:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.169833:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169835:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575119.169838:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575119.169841:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169843:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575119.169845:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575119.169848:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575119.169851:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.169852:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575119.169854:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575119.169861:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575119.169864:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575119.169867:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575119.169871:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.169874:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575119.169876:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575119.169880:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575119.169886:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575119.169892:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575119.169900:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810369152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575119.169908:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575119.169911:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.169916:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810369152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575119.169938:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575119.169941:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575119.169944:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575119.169946:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575119.169947:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575119.169949:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575119.169951:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15585 00000400:00000200:0.0:1788575119.169955:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575119.169957:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575119.169959:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575119.169962:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169964:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.169966:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.169969:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575119.169970:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575119.169972:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575119.169973:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575119.169975:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575119.169977:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575119.169989:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575119.169997:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170002:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170006:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170009:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575119.170013:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575119.170018:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170022:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575119.170025:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575119.170032:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575119.170037:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170039:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575119.170044:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575119.170050:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170053:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575119.170059:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575119.170062:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575119.170065:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575119.170067:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170069:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575119.170072:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575119.170075:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575119.170081:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575119.170088:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575119.170095:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810369280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575119.170109:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575119.170112:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.170118:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810369280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575119.170139:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575119.170148:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575119.170154:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575119.170157:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575119.170160:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575119.170165:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575119.170171:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15585 00000400:00000200:0.0:1788575119.170192:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575119.170197:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575119.170200:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575119.170204:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170207:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170211:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170215:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575119.170221:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575119.170223:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575119.170226:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575119.170229:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575119.170235:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575119.170239:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575119.170243:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170247:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170252:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170255:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575119.170259:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575119.170270:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170276:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575119.170280:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575119.170284:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575119.170288:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170293:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575119.170301:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575119.170305:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170308:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575119.170312:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575119.170316:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575119.170320:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575119.170323:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170326:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575119.170328:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575119.170331:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575119.170336:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575119.170341:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575119.170349:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810369408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575119.170363:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575119.170366:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.170375:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810369408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575119.170391:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575119.170396:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575119.170404:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575119.170407:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575119.170410:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575119.170417:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575119.170422:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15585 00000400:00000200:0.0:1788575119.170428:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575119.170434:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575119.170438:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575119.170444:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170449:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170452:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170461:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575119.170464:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575119.170471:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575119.170473:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575119.170475:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575119.170479:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575119.170483:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575119.170490:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170494:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170497:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170501:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575119.170508:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575119.170513:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170516:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575119.170520:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575119.170524:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575119.170528:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170533:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575119.170536:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575119.170541:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170559:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575119.170565:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575119.170568:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575119.170572:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575119.170575:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170580:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575119.170582:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575119.170585:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575119.170591:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575119.170598:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575119.170606:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810369536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575119.170625:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575119.170629:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.170660:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810369536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575119.170669:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575119.170675:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575119.170679:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575119.170681:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575119.170683:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575119.170687:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575119.170690:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15585 00000400:00000200:0.0:1788575119.170697:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575119.170703:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575119.170707:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575119.170714:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170717:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170721:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170726:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575119.170731:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575119.170734:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575119.170737:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575119.170742:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575119.170747:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575119.170752:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575119.170759:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170764:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575119.170768:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170772:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575119.170792:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575119.170799:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575119.170804:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575119.170808:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575119.170812:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575119.170818:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170820:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575119.170824:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575119.170829:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170833:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575119.170838:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575119.170841:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575119.170846:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575119.170849:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575119.170852:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575119.170855:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575119.170858:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575119.170864:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575119.170869:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575119.170876:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810369664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575119.170889:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575119.170893:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.170900:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810369664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575119.170917:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575119.170923:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15595) 00000100:00080000:0.0:1788575120.002092:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575120.002096:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575120.002098:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575120.002101:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575120.002103:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575120.002112:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15590 00000400:00000200:0.0:1788575120.002120:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575120.002125:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575120.002128:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575120.002131:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002134:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002136:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002138:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575120.002141:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575120.002143:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575120.002145:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575120.002147:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575120.002149:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575120.002151:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575120.002154:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002156:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002158:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002159:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575120.002163:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575120.002165:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002167:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575120.002168:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575120.002170:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575120.002172:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002173:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575120.002175:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575120.002177:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575120.002179:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575120.002181:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575120.002182:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002183:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575120.002185:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575120.002190:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575120.002194:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575120.002196:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.002202:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810369920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575120.002210:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575120.002212:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.002225:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810369920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575120.002246:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575120.002248:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575120.002250:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575120.002251:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575120.002252:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575120.002254:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575120.002256:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15590 00000400:00000200:0.0:1788575120.002259:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575120.002261:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575120.002263:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575120.002265:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002267:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002268:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002270:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575120.002271:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575120.002273:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575120.002275:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575120.002277:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575120.002281:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575120.002291:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575120.002298:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002307:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002310:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002312:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575120.002317:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575120.002322:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002326:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575120.002330:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575120.002336:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575120.002340:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002342:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575120.002347:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575120.002350:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002353:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575120.002360:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575120.002365:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575120.002370:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575120.002373:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002378:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575120.002381:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575120.002384:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575120.002389:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575120.002394:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.002402:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810370048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575120.002416:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575120.002420:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.002429:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810370048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575120.002446:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575120.002451:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575120.002457:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575120.002466:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575120.002469:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575120.002471:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575120.002477:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15590 00000400:00000200:0.0:1788575120.002484:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575120.002492:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575120.002495:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575120.002501:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002506:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002511:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002516:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575120.002521:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575120.002524:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575120.002528:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575120.002532:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575120.002535:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575120.002538:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575120.002546:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002548:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002549:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002553:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575120.002557:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575120.002574:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002577:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575120.002580:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575120.002582:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575120.002586:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002588:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575120.002590:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575120.002592:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002600:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575120.002603:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575120.002607:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575120.002611:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575120.002614:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002616:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575120.002618:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575120.002621:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575120.002625:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575120.002631:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.002668:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810370176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575120.002674:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575120.002676:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.002681:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810370176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575120.002700:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575120.002706:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575120.002714:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575120.002718:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575120.002722:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575120.002727:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575120.002733:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15590 00000400:00000200:0.0:1788575120.002742:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575120.002748:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575120.002753:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575120.002760:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002766:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002772:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002778:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575120.002783:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575120.002792:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575120.002795:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575120.002799:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575120.002806:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575120.002812:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575120.002818:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002823:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002827:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002830:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575120.002834:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575120.002840:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002842:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575120.002845:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575120.002849:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575120.002853:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002857:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575120.002860:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575120.002863:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002866:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575120.002870:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575120.002874:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575120.002878:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575120.002881:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.002883:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575120.002885:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575120.002888:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575120.002895:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575120.002900:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.002906:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810370304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575120.002914:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575120.002916:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.002925:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810370304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575120.002941:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575120.002944:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575120.002948:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575120.002951:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575120.002953:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575120.002955:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575120.002957:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15590 00000400:00000200:0.0:1788575120.002964:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575120.002971:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575120.002976:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575120.002983:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002989:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.002994:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.002999:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575120.003004:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575120.003008:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575120.003010:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575120.003013:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575120.003017:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575120.003019:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575120.003022:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.003025:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575120.003029:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575120.003033:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575120.003036:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575120.003041:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575120.003045:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575120.003054:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575120.003058:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575120.003062:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.003064:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575120.003065:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575120.003067:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.003071:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575120.003075:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575120.003078:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575120.003081:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575120.003084:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.003087:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575120.003089:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575120.003092:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575120.003098:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575120.003102:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.003109:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810370432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575120.003125:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575120.003129:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.003137:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810370432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575120.003156:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575120.003162:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15596) 00000020:00000001:0.0:1788575122.625834:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575122.625837:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575122.625839:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575123.841719:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310262815, at 4310262848 00000100:00000010:0.0:1788575123.841734:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215420. 00000100:00080000:0.0:1788575125.313756:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575125.313766:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575125.313768:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575125.313787:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575125.313791:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575125.313799:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15591 00000400:00000200:0.0:1788575125.313808:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575125.313813:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575125.313816:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575125.313819:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313822:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.313824:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313826:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575125.313830:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575125.313832:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575125.313834:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575125.313838:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575125.313841:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575125.313843:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575125.313845:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313847:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.313849:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313851:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575125.313854:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575125.313857:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313859:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575125.313863:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575125.313865:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575125.313867:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.313869:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575125.313871:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575125.313876:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575125.313879:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575125.313882:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575125.313887:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.313889:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575125.313891:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575125.313894:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575125.313899:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575125.313904:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.313911:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810370560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575125.313918:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575125.313921:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.313924:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810370560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575125.313941:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575125.313943:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575125.313945:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575125.313946:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575125.313947:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575125.313949:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575125.313950:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15591 00000400:00000200:0.0:1788575125.313953:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575125.313955:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575125.313957:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575125.313959:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313961:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.313963:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313965:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575125.313967:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575125.313968:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575125.313970:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575125.313973:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575125.313975:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575125.313982:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575125.313986:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313991:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.313995:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.313999:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575125.314003:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575125.314008:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314014:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575125.314018:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575125.314026:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575125.314030:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314032:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575125.314037:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575125.314041:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314044:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575125.314048:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575125.314053:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575125.314056:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575125.314058:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314060:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575125.314061:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575125.314063:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575125.314067:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575125.314073:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.314079:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810370688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575125.314090:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575125.314092:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.314098:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810370688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575125.314108:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575125.314115:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575125.314118:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575125.314121:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575125.314123:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575125.314127:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575125.314131:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15591 00000400:00000200:0.0:1788575125.314136:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575125.314139:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575125.314143:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575125.314156:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314159:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314161:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314164:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575125.314166:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575125.314167:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575125.314168:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575125.314169:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575125.314170:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575125.314174:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575125.314178:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314188:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314196:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314201:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575125.314206:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575125.314212:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314218:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575125.314222:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575125.314228:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575125.314233:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314238:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575125.314247:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575125.314253:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314257:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575125.314263:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575125.314269:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575125.314273:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575125.314275:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314278:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575125.314280:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575125.314284:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575125.314291:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575125.314296:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.314303:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810370816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575125.314317:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575125.314321:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.314328:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810370816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575125.314342:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575125.314347:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575125.314352:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575125.314355:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575125.314359:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575125.314363:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575125.314368:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15591 00000400:00000200:0.0:1788575125.314375:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575125.314382:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575125.314387:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575125.314394:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314398:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314402:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314411:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575125.314416:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575125.314419:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575125.314422:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575125.314427:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575125.314431:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575125.314434:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575125.314441:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314445:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314448:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314452:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575125.314456:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575125.314461:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314472:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575125.314477:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575125.314483:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575125.314488:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314491:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575125.314496:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575125.314501:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314505:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575125.314510:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575125.314515:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575125.314521:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575125.314523:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314528:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575125.314532:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575125.314537:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575125.314545:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575125.314551:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.314561:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810370944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575125.314602:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575125.314607:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.314618:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810370944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575125.314630:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575125.314669:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575125.314674:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575125.314676:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575125.314677:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575125.314680:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575125.314683:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15591 00000400:00000200:0.0:1788575125.314690:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575125.314698:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575125.314702:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575125.314705:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314710:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314714:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314719:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575125.314725:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575125.314728:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575125.314732:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575125.314735:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575125.314739:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575125.314744:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575125.314752:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314754:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575125.314756:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314758:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575125.314769:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575125.314775:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575125.314781:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575125.314785:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575125.314790:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575125.314795:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314799:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575125.314802:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575125.314805:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314807:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575125.314812:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575125.314817:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575125.314822:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575125.314825:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.314829:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575125.314833:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575125.314837:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575125.314844:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575125.314850:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.314859:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810371072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575125.314873:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575125.314878:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.314886:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810371072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575125.314897:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575125.314904:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15602) 00000020:00000001:0.0:1788575127.745881:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575127.745885:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575127.745888:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575129.153752:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310268126, at 4310268160 00000100:00000010:0.0:1788575129.153768:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215180. 00000100:00080000:0.0:1788575130.433787:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575130.433797:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575130.433799:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575130.433803:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575130.433807:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575130.433814:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15597 00000400:00000200:0.0:1788575130.433823:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575130.433831:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575130.433835:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575130.433838:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.433841:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.433844:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.433846:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575130.433852:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575130.433854:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575130.433857:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575130.433861:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575130.433864:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575130.433872:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575130.433875:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.433877:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.433880:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.433882:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575130.433884:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575130.433887:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575130.433889:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575130.433891:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575130.433893:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575130.433896:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.433897:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575130.433921:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575130.433929:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575130.433933:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575130.433937:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575130.433939:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.433942:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575130.433946:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575130.433951:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575130.433960:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575130.433967:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.433977:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810371200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575130.433985:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575130.433989:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.433992:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810371200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575130.434014:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575130.434017:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575130.434021:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575130.434022:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575130.434024:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575130.434026:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575130.434029:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15597 00000400:00000200:0.0:1788575130.434034:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575130.434037:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575130.434041:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575130.434044:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434046:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434049:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434051:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575130.434054:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575130.434062:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575130.434064:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575130.434067:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575130.434069:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575130.434071:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575130.434075:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434078:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434081:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434086:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575130.434090:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575130.434094:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434099:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575130.434100:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575130.434107:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575130.434111:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434113:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575130.434119:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575130.434125:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434129:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575130.434136:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575130.434140:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575130.434146:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575130.434147:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434151:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575130.434152:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575130.434155:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575130.434166:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575130.434176:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.434192:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810371328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575130.434219:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575130.434222:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.434241:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810371328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575130.434256:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575130.434260:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575130.434266:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575130.434269:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575130.434272:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575130.434280:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575130.434283:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15597 00000400:00000200:0.0:1788575130.434289:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575130.434294:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575130.434304:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575130.434316:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434319:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434323:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434331:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575130.434348:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575130.434354:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575130.434360:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575130.434365:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575130.434369:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575130.434373:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575130.434379:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434384:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434403:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434412:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575130.434424:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575130.434437:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434451:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575130.434464:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575130.434471:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575130.434480:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434487:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575130.434494:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575130.434506:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434508:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575130.434512:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575130.434515:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575130.434517:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575130.434518:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434519:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575130.434520:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575130.434522:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575130.434531:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575130.434537:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.434547:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810371456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575130.434557:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575130.434561:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.434566:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810371456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575130.434579:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575130.434583:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575130.434591:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575130.434595:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575130.434599:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575130.434607:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575130.434614:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15597 00000400:00000200:0.0:1788575130.434620:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575130.434628:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575130.434664:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575130.434673:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434675:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434678:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434686:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575130.434694:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575130.434695:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575130.434699:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575130.434702:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575130.434710:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575130.434718:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575130.434729:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434730:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.434749:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434752:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575130.434759:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575130.434770:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575130.434777:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575130.434787:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575130.434796:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575130.434801:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434807:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575130.434813:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575130.434819:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434822:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575130.434829:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575130.434835:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575130.434842:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575130.434846:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.434851:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575130.434854:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575130.434859:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575130.434868:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575130.434885:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.434894:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810371584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575130.434922:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575130.434932:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.434944:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810371584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575130.434973:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575130.434981:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575130.434994:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575130.435000:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575130.435006:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575130.435013:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575130.435029:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15597 00000400:00000200:0.0:1788575130.435041:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575130.435053:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575130.435061:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575130.435071:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.435077:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.435080:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.435089:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575130.435097:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575130.435102:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575130.435109:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575130.435116:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575130.435128:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575130.435139:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575130.435151:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.435160:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575130.435175:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575130.435182:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575130.435189:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575130.435198:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575130.435208:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575130.435220:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575130.435228:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575130.435239:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.435246:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575130.435251:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575130.435258:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.435263:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575130.435272:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575130.435277:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575130.435283:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575130.435289:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575130.435296:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575130.435300:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575130.435306:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575130.435315:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575130.435324:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.435336:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810371712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575130.435369:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575130.435375:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.435385:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810371712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575130.435403:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575130.435409:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15607) 00000020:00000001:0.0:1788575132.865860:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575132.865864:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575132.865868:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575134.273748:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310273246, at 4310273280 00000100:00000010:0.0:1788575134.273762:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215180. 00000020:00000001:0.0:1788575137.985858:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575137.985861:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575137.985864:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575139.393717:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310278367, at 4310278400 00000100:00000010:0.0:1788575139.393739:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000800:00000200:0.0:1788575141.121922:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575143.105866:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575143.105869:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575143.105872:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575145.409725:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310284383, at 4310284416 00000100:00000010:0.0:1788575145.409747:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00080000:0.0:1788575146.817775:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575146.817784:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575146.817785:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575146.817788:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575146.817791:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575146.817796:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15613 00000400:00000200:0.0:1788575146.817803:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575146.817808:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575146.817810:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575146.817812:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817814:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.817816:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817817:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575146.817820:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575146.817821:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575146.817823:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575146.817826:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575146.817827:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575146.817842:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575146.817845:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817846:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.817847:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817848:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575146.817850:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575146.817852:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817854:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575146.817856:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575146.817858:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575146.817860:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.817861:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575146.817863:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575146.817867:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575146.817869:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575146.817872:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575146.817873:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.817875:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575146.817877:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575146.817881:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575146.817885:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575146.817890:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.817897:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810373888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575146.817903:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575146.817907:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.817910:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810373888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575146.817930:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575146.817932:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575146.817935:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575146.817939:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575146.817940:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575146.817942:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575146.817943:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15613 00000400:00000200:0.0:1788575146.817949:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575146.817952:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575146.817953:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575146.817955:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817956:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.817958:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817959:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575146.817961:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575146.817962:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575146.817963:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575146.817964:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575146.817965:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575146.817967:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575146.817968:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817970:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.817971:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817972:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575146.817974:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575146.817980:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575146.817984:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575146.817988:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575146.817993:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575146.817997:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818001:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575146.818004:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575146.818010:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818016:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575146.818020:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575146.818023:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575146.818029:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575146.818031:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818034:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575146.818036:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575146.818038:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575146.818043:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575146.818046:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.818052:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810374016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575146.818063:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575146.818065:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.818067:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810374016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575146.818079:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575146.818082:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575146.818086:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575146.818089:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575146.818091:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575146.818093:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575146.818097:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15613 00000400:00000200:0.0:1788575146.818101:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575146.818106:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575146.818110:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575146.818114:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818116:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818119:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818123:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575146.818127:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575146.818136:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575146.818139:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575146.818142:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575146.818146:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575146.818149:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575146.818152:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818156:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818159:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818161:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575146.818164:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575146.818169:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818173:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575146.818175:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575146.818179:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575146.818183:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818201:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575146.818203:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575146.818204:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818205:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575146.818208:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575146.818211:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575146.818213:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575146.818216:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818219:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575146.818221:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575146.818223:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575146.818235:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575146.818240:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.818246:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810374144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575146.818254:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575146.818256:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.818266:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810374144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575146.818276:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575146.818281:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575146.818286:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575146.818290:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575146.818292:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575146.818297:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575146.818301:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15613 00000400:00000200:0.0:1788575146.818306:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575146.818309:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575146.818313:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575146.818319:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818322:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818326:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818327:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575146.818329:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575146.818332:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575146.818334:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575146.818336:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575146.818338:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575146.818341:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575146.818345:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818347:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818350:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818354:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575146.818357:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575146.818361:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818366:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575146.818371:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575146.818374:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575146.818376:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818377:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575146.818379:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575146.818381:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818383:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575146.818385:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575146.818389:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575146.818393:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575146.818396:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818399:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575146.818401:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575146.818404:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575146.818407:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575146.818411:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.818417:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810374272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575146.818427:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575146.818429:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.818434:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810374272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575146.818440:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575146.818443:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575146.818445:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575146.818446:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575146.818447:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575146.818449:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575146.818453:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15613 00000400:00000200:0.0:1788575146.818457:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575146.818460:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575146.818463:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575146.818475:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818481:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818486:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818489:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575146.818495:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575146.818498:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575146.818502:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575146.818507:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575146.818512:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575146.818518:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575146.818523:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818528:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575146.818532:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818535:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575146.818537:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575146.818543:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575146.818549:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575146.818552:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575146.818556:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575146.818559:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818563:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575146.818565:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575146.818570:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818573:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575146.818579:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575146.818583:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575146.818588:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575146.818590:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575146.818594:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575146.818597:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575146.818600:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575146.818604:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575146.818614:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.818621:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810374400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575146.818626:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575146.818627:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.818630:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810374400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575146.818664:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575146.818669:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15623) 00000020:00000001:0.0:1788575148.225849:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575148.225853:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575148.225856:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575150.657779:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310289630, at 4310289664 00000100:00000010:0.0:1788575150.657799:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00080000:0.0:1788575151.937726:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575151.937744:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575151.937746:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575151.937749:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575151.937753:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575151.937758:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15618 00000400:00000200:0.0:1788575151.937765:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575151.937770:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575151.937773:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575151.937775:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937777:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.937779:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937781:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575151.937784:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575151.937786:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575151.937788:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575151.937812:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575151.937814:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575151.937816:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575151.937818:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937820:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.937821:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937823:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575151.937825:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575151.937827:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937829:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575151.937831:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575151.937834:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575151.937836:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.937837:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575151.937839:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575151.937843:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575151.937846:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575151.937849:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575151.937850:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.937852:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575151.937855:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575151.937859:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575151.937865:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575151.937870:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.937890:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810374528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575151.937899:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575151.937902:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.937906:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810374528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575151.937925:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575151.937927:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575151.937934:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575151.937935:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575151.937936:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575151.937938:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575151.937940:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15618 00000400:00000200:0.0:1788575151.937943:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575151.937946:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575151.937948:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575151.937950:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937952:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.937954:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937956:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575151.937958:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575151.937959:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575151.937960:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575151.937962:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575151.937964:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575151.937967:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575151.937971:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937974:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.937979:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937981:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575151.937986:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575151.937990:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575151.937995:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575151.937999:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575151.938006:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575151.938009:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938012:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575151.938017:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575151.938027:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938031:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575151.938035:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575151.938039:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575151.938044:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575151.938045:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938047:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575151.938050:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575151.938052:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575151.938058:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575151.938064:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.938072:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810374656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575151.938080:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575151.938084:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.938089:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810374656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575151.938100:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575151.938104:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575151.938109:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575151.938113:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575151.938117:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575151.938120:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575151.938125:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15618 00000400:00000200:0.0:1788575151.938133:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575151.938140:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575151.938144:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575151.938149:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938155:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938162:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938174:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575151.938181:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575151.938185:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575151.938197:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575151.938202:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575151.938207:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575151.938212:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575151.938221:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938226:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938230:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938235:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575151.938241:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575151.938248:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938253:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575151.938256:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575151.938259:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575151.938261:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938263:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575151.938266:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575151.938269:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938275:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575151.938287:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575151.938293:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575151.938296:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575151.938300:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938303:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575151.938306:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575151.938310:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575151.938318:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575151.938324:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.938333:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810374784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575151.938362:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575151.938377:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.938385:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810374784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575151.938404:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575151.938407:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575151.938413:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575151.938418:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575151.938422:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575151.938427:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575151.938434:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15618 00000400:00000200:0.0:1788575151.938446:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575151.938455:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575151.938461:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575151.938470:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938475:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938483:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938489:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575151.938496:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575151.938499:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575151.938504:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575151.938507:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575151.938512:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575151.938518:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575151.938524:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938526:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938528:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938533:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575151.938553:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575151.938563:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938570:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575151.938576:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575151.938584:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575151.938589:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938594:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575151.938597:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575151.938600:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938602:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575151.938606:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575151.938612:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575151.938617:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575151.938620:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938625:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575151.938628:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575151.938632:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575151.938665:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575151.938673:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.938685:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810374912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575151.938700:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575151.938706:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.938716:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810374912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575151.938736:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575151.938740:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575151.938746:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575151.938752:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575151.938757:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575151.938763:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575151.938777:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15618 00000400:00000200:0.0:1788575151.938787:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575151.938797:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575151.938802:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575151.938809:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938814:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938819:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938823:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575151.938828:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575151.938831:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575151.938834:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575151.938838:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575151.938842:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575151.938845:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575151.938852:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938857:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575151.938862:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938868:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575151.938874:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575151.938893:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575151.938901:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575151.938907:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575151.938912:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575151.938918:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938923:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575151.938928:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575151.938934:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938939:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575151.938945:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575151.938951:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575151.938957:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575151.938967:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.938973:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575151.938975:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575151.938976:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575151.938980:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575151.938985:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.938994:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810375040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575151.939009:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575151.939013:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.939019:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810375040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575151.939034:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575151.939040:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15628) 00000020:00000001:0.0:1788575153.345898:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575153.345902:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575153.345904:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575155.777757:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310294750, at 4310294784 00000100:00000010:0.0:1788575155.777777:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788575158.465779:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575158.465782:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575158.465785:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575160.897697:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310299870, at 4310299904 00000100:00000010:0.0:1788575160.897713:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00080000:0.0:1788575162.177750:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575162.177758:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575162.177760:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575162.177763:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575162.177767:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575162.177773:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15628 00000400:00000200:0.0:1788575162.177780:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575162.177785:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575162.177803:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575162.177805:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177808:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.177809:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177811:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575162.177814:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575162.177815:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575162.177817:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575162.177820:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575162.177822:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575162.177823:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575162.177825:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177827:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.177828:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177830:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575162.177832:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575162.177834:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177836:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575162.177838:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575162.177841:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575162.177843:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.177844:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575162.177846:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575162.177851:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575162.177853:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575162.177856:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575162.177857:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.177860:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575162.177861:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575162.177865:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575162.177870:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575162.177876:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575162.177887:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810375808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575162.177895:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575162.177898:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.177901:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810375808/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575162.177919:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575162.177922:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575162.177924:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575162.177925:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575162.177927:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575162.177929:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575162.177931:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15628 00000400:00000200:0.0:1788575162.177933:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575162.177936:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575162.177937:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575162.177939:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177940:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.177941:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177943:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575162.177945:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575162.177946:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575162.177947:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575162.177948:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575162.177950:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575162.177952:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575162.177954:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177955:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.177957:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177963:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575162.177967:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575162.177970:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575162.177974:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575162.177977:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575162.177982:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575162.177985:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.177988:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575162.177992:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575162.177997:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178000:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575162.178005:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575162.178008:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575162.178012:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575162.178014:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178016:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575162.178018:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575162.178023:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575162.178027:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575162.178034:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575162.178051:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810375936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575162.178069:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575162.178073:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.178080:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810375936/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575162.178089:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575162.178094:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575162.178100:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575162.178104:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575162.178108:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575162.178121:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575162.178128:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15628 00000400:00000200:0.0:1788575162.178138:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575162.178143:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575162.178146:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575162.178154:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178160:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178167:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178173:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575162.178178:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575162.178182:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575162.178186:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575162.178191:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575162.178197:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575162.178199:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575162.178206:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178210:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178214:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178217:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575162.178221:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575162.178226:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178228:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575162.178230:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575162.178234:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575162.178238:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178240:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575162.178243:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575162.178247:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178249:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575162.178255:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575162.178279:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575162.178281:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575162.178285:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178288:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575162.178290:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575162.178292:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575162.178298:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575162.178303:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575162.178311:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810376064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575162.178328:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575162.178331:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.178338:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810376064/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575162.178356:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575162.178359:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575162.178363:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575162.178366:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575162.178368:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575162.178373:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575162.178378:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15628 00000400:00000200:0.0:1788575162.178385:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575162.178389:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575162.178391:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575162.178393:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178396:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178400:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178403:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575162.178406:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575162.178411:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575162.178421:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575162.178427:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575162.178434:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575162.178441:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575162.178451:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178457:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178462:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178466:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575162.178470:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575162.178474:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178480:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575162.178484:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575162.178489:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575162.178493:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178498:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575162.178501:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575162.178505:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178508:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575162.178512:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575162.178518:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575162.178521:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575162.178523:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178527:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575162.178529:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575162.178532:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575162.178536:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575162.178542:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575162.178551:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810376192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575162.178567:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575162.178570:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.178576:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810376192/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575162.178610:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575162.178617:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575162.178625:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575162.178629:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575162.178633:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575162.178683:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575162.178689:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15628 00000400:00000200:0.0:1788575162.178696:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575162.178701:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575162.178704:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575162.178709:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178714:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178719:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178723:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575162.178727:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575162.178728:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575162.178729:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575162.178731:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575162.178733:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575162.178736:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575162.178741:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178744:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575162.178748:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178752:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575162.178757:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575162.178762:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575162.178767:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575162.178775:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575162.178779:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575162.178784:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178787:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575162.178791:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575162.178794:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178797:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575162.178802:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575162.178805:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575162.178809:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575162.178812:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575162.178815:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575162.178818:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575162.178821:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575162.178827:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575162.178833:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575162.178841:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810376320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575162.178861:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575162.178865:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.178870:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810376320/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575162.178885:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575162.178890:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15638) 00000100:00080000:0.0:1788575163.330404:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575163.330409:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575163.330411:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575163.330414:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575163.330417:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575163.330420:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15633 00000400:00000200:0.0:1788575163.330426:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575163.330430:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575163.330438:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575163.330441:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330443:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330445:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330447:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575163.330449:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575163.330450:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575163.330452:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575163.330454:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575163.330455:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575163.330457:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575163.330459:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330460:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330462:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330463:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575163.330465:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575163.330467:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330469:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575163.330470:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575163.330472:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575163.330474:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.330476:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575163.330477:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575163.330480:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575163.330482:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575163.330484:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575163.330485:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.330487:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575163.330488:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575163.330490:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575163.330494:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575163.330496:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.330505:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810376576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575163.330512:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575163.330514:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.330517:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810376576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575163.330596:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575163.330600:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575163.330604:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575163.330608:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575163.330610:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575163.330615:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575163.330619:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15633 00000400:00000200:0.0:1788575163.330625:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575163.330631:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575163.330681:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575163.330688:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330693:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330698:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330703:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575163.330707:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575163.330710:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575163.330713:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575163.330718:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575163.330721:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575163.330723:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575163.330727:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330730:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330735:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330745:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575163.330748:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575163.330754:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330759:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575163.330762:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575163.330768:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575163.330771:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.330776:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575163.330781:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575163.330785:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.330788:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575163.330792:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575163.330795:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575163.330797:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575163.330800:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.330802:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575163.330805:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575163.330808:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575163.330814:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575163.330820:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.330830:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810376704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575163.330847:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575163.330852:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.330860:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810376704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575163.330881:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575163.330885:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575163.330891:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575163.330893:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575163.330896:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575163.330905:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575163.330910:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15633 00000400:00000200:0.0:1788575163.330915:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575163.330920:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575163.330923:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575163.330929:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330934:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330938:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330943:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575163.330947:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575163.330952:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575163.330956:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575163.330960:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575163.330964:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575163.330970:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575163.330977:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330983:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.330987:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.330990:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575163.330993:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575163.331000:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331005:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575163.331008:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575163.331010:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575163.331014:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331016:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575163.331018:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575163.331022:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331026:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575163.331032:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575163.331043:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575163.331050:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575163.331053:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331059:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575163.331062:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575163.331066:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575163.331073:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575163.331079:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.331087:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810376832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575163.331095:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575163.331099:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.331103:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810376832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575163.331112:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575163.331116:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575163.331121:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575163.331124:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575163.331126:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575163.331130:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575163.331134:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15633 00000400:00000200:0.0:1788575163.331139:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575163.331143:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575163.331147:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575163.331152:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331157:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.331160:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331164:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575163.331168:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575163.331170:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575163.331178:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575163.331181:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575163.331186:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575163.331203:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575163.331211:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331215:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.331219:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331221:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575163.331226:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575163.331231:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331236:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575163.331239:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575163.331243:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575163.331248:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331250:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575163.331254:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575163.331259:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331262:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575163.331268:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575163.331271:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575163.331274:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575163.331277:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331279:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575163.331282:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575163.331285:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575163.331289:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575163.331292:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.331302:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810376960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575163.331321:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575163.331324:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.331330:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810376960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575163.331347:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575163.331350:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575163.331355:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575163.331358:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575163.331360:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575163.331365:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575163.331370:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15633 00000400:00000200:0.0:1788575163.331376:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575163.331381:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575163.331385:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575163.331390:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331394:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.331397:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331399:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575163.331401:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575163.331402:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575163.331403:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575163.331405:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575163.331408:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575163.331411:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575163.331416:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331419:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575163.331423:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331428:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575163.331430:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575163.331432:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575163.331434:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575163.331438:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575163.331441:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575163.331444:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331447:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575163.331450:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575163.331456:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331460:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575163.331465:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575163.331468:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575163.331471:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575163.331474:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.331478:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575163.331481:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575163.331483:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575163.331485:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575163.331488:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.331493:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810377088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575163.331503:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575163.331507:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.331515:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810377088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575163.331528:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575163.331532:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15640) 00000020:00000001:0.0:1788575163.585833:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575163.585836:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575163.585838:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575167.169755:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310306143, at 4310306176 00000100:00000010:0.0:1788575167.169777:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788575168.705832:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575168.705837:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575168.705841:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575172.673736:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310311646, at 4310311680 00000100:00000010:0.0:1788575172.673782:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788575173.825883:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575173.825887:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575173.825890:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575177.793752:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310316766, at 4310316800 00000100:00000010:0.0:1788575177.793769:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000020:00000001:0.0:1788575178.945842:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575178.945845:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575178.945848:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575179.009924:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788575179.073751:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575179.073764:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575179.073766:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575179.073772:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575179.073776:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575179.073785:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15645 00000400:00000200:0.0:1788575179.073797:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575179.073804:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575179.073808:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575179.073811:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.073815:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.073818:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.073821:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575179.073826:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575179.073828:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575179.073832:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575179.073836:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575179.073838:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575179.073840:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575179.073843:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.073845:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.073867:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.073869:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575179.073873:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575179.073876:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575179.073879:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575179.073882:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575179.073886:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575179.073889:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.073891:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575179.073893:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575179.073901:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575179.073905:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575179.073909:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575179.073910:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.073914:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575179.073916:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575179.073922:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575179.073930:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575179.073938:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.073948:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810378496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575179.073958:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575179.073964:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.073968:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810378496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575179.073997:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575179.074001:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575179.074004:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575179.074006:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575179.074007:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575179.074010:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575179.074018:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15645 00000400:00000200:0.0:1788575179.074023:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575179.074026:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575179.074028:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575179.074031:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074032:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074034:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074036:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575179.074038:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575179.074039:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575179.074040:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575179.074042:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575179.074044:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575179.074047:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575179.074050:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074054:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074056:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074060:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575179.074062:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575179.074066:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074067:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575179.074069:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575179.074075:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575179.074079:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074082:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575179.074087:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575179.074091:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074094:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575179.074097:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575179.074100:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575179.074105:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575179.074111:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074114:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575179.074116:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575179.074119:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575179.074123:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575179.074129:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.074136:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810378624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575179.074144:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575179.074146:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.074151:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810378624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575179.074166:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575179.074172:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575179.074177:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575179.074178:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575179.074182:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575179.074187:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575179.074192:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15645 00000400:00000200:0.0:1788575179.074199:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575179.074204:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575179.074207:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575179.074214:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074219:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074224:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074227:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575179.074229:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575179.074230:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575179.074231:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575179.074232:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575179.074234:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575179.074251:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575179.074257:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074260:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074266:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074271:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575179.074274:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575179.074280:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074285:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575179.074287:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575179.074291:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575179.074296:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074298:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575179.074300:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575179.074305:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074309:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575179.074314:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575179.074317:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575179.074322:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575179.074324:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074327:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575179.074330:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575179.074334:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575179.074341:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575179.074348:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.074357:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810378752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575179.074376:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575179.074380:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.074386:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810378752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575179.074406:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575179.074415:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575179.074419:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575179.074423:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575179.074424:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575179.074429:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575179.074435:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15645 00000400:00000200:0.0:1788575179.074441:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575179.074448:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575179.074453:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575179.074459:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074463:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074467:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074470:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575179.074473:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575179.074476:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575179.074478:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575179.074482:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575179.074486:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575179.074489:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575179.074495:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074499:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074503:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074507:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575179.074509:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575179.074513:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074519:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575179.074523:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575179.074527:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575179.074534:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074539:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575179.074548:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575179.074553:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074558:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575179.074562:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575179.074565:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575179.074569:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575179.074571:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074576:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575179.074579:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575179.074582:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575179.074588:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575179.074595:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.074604:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810378880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575179.074616:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575179.074620:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.074628:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810378880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575179.074673:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575179.074679:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575179.074684:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575179.074688:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575179.074691:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575179.074696:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575179.074702:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15645 00000400:00000200:0.0:1788575179.074711:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575179.074718:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575179.074723:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575179.074731:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074735:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074740:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074751:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575179.074756:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575179.074760:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575179.074763:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575179.074766:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575179.074770:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575179.074775:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575179.074784:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074789:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575179.074794:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074799:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575179.074803:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575179.074806:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575179.074809:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575179.074813:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575179.074818:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575179.074823:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074827:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575179.074831:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575179.074836:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074842:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575179.074848:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575179.074854:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575179.074859:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575179.074863:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575179.074867:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575179.074870:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575179.074875:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575179.074882:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575179.074889:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.074897:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810379008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575179.074933:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575179.074938:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.074948:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810379008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575179.074967:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.074975:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15655) 00000100:00000040:0.0:1788575182.913746:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310321886, at 4310321920 00000100:00000010:0.0:1788575182.913768:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000020:00000001:0.0:1788575184.065897:0:237084:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575184.065902:0:237084:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575184.065905:0:237084:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575184.193748:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575184.193761:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575184.193763:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575184.193768:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575184.193773:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575184.193781:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15650 00000400:00000200:0.0:1788575184.193791:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575184.193799:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575184.193802:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575184.193805:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.193808:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.193811:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.193813:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575184.193818:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575184.193820:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575184.193823:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575184.193826:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575184.193828:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575184.193856:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575184.193859:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.193861:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.193863:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.193864:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575184.193867:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575184.193870:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575184.193872:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575184.193875:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575184.193879:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575184.193881:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.193883:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575184.193885:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575184.193894:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575184.193898:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575184.193902:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575184.193904:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.193907:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575184.193909:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575184.193915:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575184.193923:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575184.193942:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.193952:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810379136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575184.193963:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575184.193968:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.193973:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810379136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575184.193996:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575184.193998:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575184.194001:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575184.194006:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575184.194008:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575184.194010:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575184.194012:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15650 00000400:00000200:0.0:1788575184.194016:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575184.194019:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575184.194020:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575184.194022:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194024:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194029:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194030:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575184.194034:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575184.194035:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575184.194038:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575184.194040:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575184.194041:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575184.194043:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575184.194050:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194054:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194057:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194060:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575184.194064:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575184.194066:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194069:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575184.194073:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575184.194081:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575184.194084:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194086:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575184.194091:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575184.194095:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194103:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575184.194107:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575184.194111:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575184.194114:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575184.194116:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194120:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575184.194121:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575184.194122:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575184.194126:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575184.194130:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.194134:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810379264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575184.194146:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575184.194148:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.194155:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810379264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575184.194169:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575184.194174:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575184.194181:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575184.194182:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575184.194183:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575184.194185:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575184.194188:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15650 00000400:00000200:0.0:1788575184.194194:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575184.194197:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575184.194198:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575184.194200:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194202:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194203:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194205:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575184.194210:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575184.194215:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575184.194219:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575184.194220:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575184.194224:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575184.194228:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575184.194233:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194236:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194240:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194243:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575184.194246:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575184.194251:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194255:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575184.194257:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575184.194259:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575184.194261:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194263:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575184.194267:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575184.194270:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194273:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575184.194278:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575184.194281:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575184.194283:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575184.194284:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194288:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575184.194288:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575184.194290:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575184.194295:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575184.194299:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.194305:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810379392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575184.194317:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575184.194319:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.194328:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810379392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575184.194339:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575184.194342:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575184.194346:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575184.194348:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575184.194350:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575184.194353:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575184.194356:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15650 00000400:00000200:0.0:1788575184.194363:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575184.194368:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575184.194371:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575184.194374:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194376:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194378:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194381:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575184.194384:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575184.194386:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575184.194389:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575184.194392:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575184.194395:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575184.194397:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575184.194402:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194404:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194406:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194407:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575184.194411:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575184.194415:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194419:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575184.194424:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575184.194426:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575184.194429:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194430:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575184.194432:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575184.194434:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194438:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575184.194441:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575184.194444:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575184.194446:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575184.194448:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194451:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575184.194453:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575184.194456:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575184.194461:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575184.194467:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.194477:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810379520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575184.194490:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575184.194495:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.194503:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810379520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575184.194530:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575184.194536:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575184.194543:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575184.194546:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575184.194549:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575184.194554:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575184.194557:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15650 00000400:00000200:0.0:1788575184.194565:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575184.194570:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575184.194574:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575184.194585:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194588:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194592:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194597:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575184.194601:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575184.194604:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575184.194608:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575184.194610:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575184.194612:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575184.194614:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575184.194617:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194622:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575184.194627:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194631:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575184.194660:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575184.194666:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575184.194671:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575184.194675:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575184.194678:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575184.194683:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194686:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575184.194690:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575184.194693:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194697:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575184.194702:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575184.194705:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575184.194710:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575184.194713:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575184.194717:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575184.194719:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575184.194723:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575184.194729:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575184.194742:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.194751:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810379648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575184.194765:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575184.194767:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.194771:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810379648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575184.194784:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.194790:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15660) 00000100:00000040:0.0:1788575189.505744:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310328479, at 4310328512 00000100:00000010:0.0:1788575189.505764:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00080000:0.0:1788575190.849763:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575190.849776:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575190.849779:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575190.849784:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575190.849789:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575190.849798:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15657 00000400:00000200:0.0:1788575190.849809:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575190.849816:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575190.849820:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575190.849823:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.849827:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.849830:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.849833:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575190.849838:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575190.849840:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575190.849844:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575190.849848:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575190.849851:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575190.849854:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575190.849878:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.849881:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.849883:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.849885:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575190.849889:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575190.849893:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575190.849896:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575190.849899:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575190.849903:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575190.849906:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.849908:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575190.849911:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575190.849927:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575190.849930:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575190.849934:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575190.849936:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.849939:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575190.849941:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575190.849946:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575190.849953:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575190.849961:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.849973:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810380544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575190.849984:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575190.849989:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.849994:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810380544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575190.850019:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575190.850023:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575190.850026:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575190.850028:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575190.850035:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575190.850038:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575190.850041:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15657 00000400:00000200:0.0:1788575190.850045:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575190.850049:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575190.850052:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575190.850055:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850058:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850061:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850063:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575190.850066:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575190.850067:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575190.850070:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575190.850073:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575190.850078:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575190.850089:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575190.850097:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850102:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850107:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850111:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575190.850115:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575190.850121:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850126:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575190.850131:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575190.850141:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575190.850146:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850150:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575190.850155:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575190.850163:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850166:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575190.850178:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575190.850183:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575190.850187:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575190.850191:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850194:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575190.850196:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575190.850199:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575190.850206:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575190.850211:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.850222:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810380672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575190.850243:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575190.850248:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.850258:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810380672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575190.850281:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575190.850286:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575190.850289:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575190.850291:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575190.850294:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575190.850297:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575190.850300:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15657 00000400:00000200:0.0:1788575190.850347:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575190.850356:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575190.850360:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575190.850365:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850369:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850374:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850379:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575190.850385:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575190.850394:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575190.850398:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575190.850402:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575190.850408:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575190.850415:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575190.850421:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850423:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850425:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850427:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575190.850430:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575190.850436:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850442:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575190.850447:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575190.850451:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575190.850457:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850462:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575190.850465:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575190.850470:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850474:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575190.850480:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575190.850485:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575190.850491:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575190.850494:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850498:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575190.850501:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575190.850505:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575190.850513:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575190.850521:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.850545:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810380800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575190.850559:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575190.850571:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.850586:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810380800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575190.850603:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575190.850606:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575190.850610:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575190.850612:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575190.850618:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575190.850622:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575190.850626:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15657 00000400:00000200:0.0:1788575190.850631:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575190.850634:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575190.850635:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575190.850637:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850638:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850662:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850666:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575190.850668:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575190.850671:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575190.850675:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575190.850677:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575190.850681:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575190.850683:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575190.850689:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850694:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850696:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850700:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575190.850701:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575190.850707:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850712:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575190.850722:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575190.850728:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575190.850732:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850736:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575190.850739:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575190.850742:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850744:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575190.850748:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575190.850750:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575190.850753:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575190.850755:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850756:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575190.850759:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575190.850763:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575190.850771:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575190.850776:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.850785:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810380928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575190.850810:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575190.850814:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.850820:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810380928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575190.850843:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575190.850849:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575190.850855:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575190.850859:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575190.850862:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575190.850868:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575190.850873:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15657 00000400:00000200:0.0:1788575190.850883:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575190.850890:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575190.850893:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575190.850903:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850907:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850913:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850918:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575190.850923:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575190.850927:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575190.850931:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575190.850934:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575190.850939:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575190.850942:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575190.850950:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850953:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575190.850955:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850958:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575190.850962:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575190.850968:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575190.850973:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575190.850978:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575190.850983:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575190.850988:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.850992:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575190.850995:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575190.851000:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.851002:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575190.851008:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575190.851013:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575190.851019:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575190.851022:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575190.851026:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575190.851028:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575190.851033:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575190.851039:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575190.851050:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.851058:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810381056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575190.851072:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575190.851077:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.851085:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810381056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575190.851099:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575190.851104:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15667) 00000100:00000040:0.0:1788575194.689728:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310333662, at 4310333696 00000100:00000010:0.0:1788575194.689748:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00000040:0.0:1788575199.809716:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310338782, at 4310338816 00000100:00000010:0.0:1788575199.809729:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00000040:0.0:1788575204.929719:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310343902, at 4310343936 00000100:00000010:0.0:1788575204.929736:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00000040:0.0:1788575210.625721:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310349599, at 4310349632 00000100:00000010:0.0:1788575210.625732:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00000040:0.0:1788575215.681721:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310354654, at 4310354688 00000100:00000010:0.0:1788575215.681736:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788575216.961795:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575216.961808:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575216.961812:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575216.961817:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575216.961821:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575216.961830:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15683 00000400:00000200:0.0:1788575216.961840:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575216.961854:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575216.961857:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575216.961860:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.961879:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.961881:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.961883:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575216.961886:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575216.961887:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575216.961889:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575216.961893:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575216.961894:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575216.961896:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575216.961898:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.961900:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.961901:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.961902:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575216.961904:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575216.961907:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575216.961908:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575216.961911:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575216.961914:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575216.961916:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.961917:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575216.961919:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575216.961924:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575216.961927:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575216.961930:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575216.961931:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.961934:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575216.961936:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575216.961939:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575216.961946:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575216.961952:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575216.961959:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810384512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575216.961975:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575216.961978:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.961982:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810384512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575216.962000:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575216.962003:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575216.962005:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575216.962007:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575216.962008:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575216.962010:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575216.962011:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15683 00000400:00000200:0.0:1788575216.962014:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575216.962017:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575216.962018:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575216.962020:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962022:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962023:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962025:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575216.962026:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575216.962028:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575216.962029:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575216.962031:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575216.962032:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575216.962034:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575216.962037:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962038:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962040:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962041:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575216.962047:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575216.962049:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962052:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575216.962056:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575216.962062:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575216.962066:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962069:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575216.962073:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575216.962076:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962080:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575216.962084:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575216.962088:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575216.962092:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575216.962094:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962097:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575216.962100:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575216.962102:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575216.962107:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575216.962113:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575216.962119:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810384640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575216.962132:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575216.962135:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.962140:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810384640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575216.962153:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575216.962157:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575216.962161:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575216.962165:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575216.962167:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575216.962171:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575216.962178:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15683 00000400:00000200:0.0:1788575216.962182:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575216.962184:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575216.962186:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575216.962188:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962189:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962193:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962199:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575216.962200:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575216.962201:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575216.962204:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575216.962206:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575216.962211:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575216.962213:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575216.962215:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962219:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962222:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962223:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575216.962225:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575216.962228:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962231:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575216.962234:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575216.962236:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575216.962238:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962240:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575216.962243:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575216.962247:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962251:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575216.962255:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575216.962259:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575216.962277:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575216.962284:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962289:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575216.962291:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575216.962293:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575216.962298:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575216.962304:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575216.962311:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810384768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575216.962321:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575216.962324:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.962328:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810384768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575216.962339:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575216.962341:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575216.962345:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575216.962348:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575216.962352:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575216.962357:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575216.962363:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15683 00000400:00000200:0.0:1788575216.962411:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575216.962419:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575216.962423:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575216.962429:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962433:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962438:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962443:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575216.962447:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575216.962450:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575216.962452:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575216.962457:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575216.962462:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575216.962475:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575216.962481:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962485:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962488:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962494:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575216.962499:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575216.962507:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962510:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575216.962513:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575216.962519:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575216.962523:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962526:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575216.962531:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575216.962546:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962549:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575216.962555:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575216.962559:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575216.962563:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575216.962566:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962570:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575216.962572:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575216.962575:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575216.962579:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575216.962585:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575216.962591:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810384896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575216.962602:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575216.962606:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.962610:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810384896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575216.962619:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575216.962628:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575216.962657:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575216.962658:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575216.962660:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575216.962662:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575216.962664:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15683 00000400:00000200:0.0:1788575216.962668:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575216.962671:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575216.962673:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575216.962675:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962677:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962678:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962680:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575216.962681:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575216.962682:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575216.962683:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575216.962685:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575216.962687:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575216.962689:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575216.962695:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962697:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575216.962701:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962704:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575216.962707:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575216.962711:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575216.962714:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575216.962717:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575216.962722:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575216.962726:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962728:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575216.962736:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575216.962742:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962748:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575216.962756:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575216.962761:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575216.962768:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575216.962772:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575216.962775:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575216.962780:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575216.962784:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575216.962792:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575216.962799:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575216.962810:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810385024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575216.962837:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575216.962842:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.962853:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810385024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575216.962865:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575216.962871:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15693) 00000800:00000200:0.0:1788575217.921951:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575220.801724:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310359774, at 4310359808 00000100:00000010:0.0:1788575220.801743:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00000040:0.0:1788575225.921713:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310364894, at 4310364928 00000100:00000010:0.0:1788575225.921729:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 10000000:00000001:0.0:1788575227.201768:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575227.201791:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9a00. 00000020:00000010:0.0:1788575227.201799:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575227.201807:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575227.201813:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575227.201820:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575227.201828:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575227.201853:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575227.201861:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788575227.201868:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985dc00. 10000000:00000040:0.0:1788575227.201875:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575227.201878:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575227.201885:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575227.201892:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575227.201896:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575227.201902:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575227.201906:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575227.201909:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788575227.201915:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985dc00. 00000020:00000010:0.0:1788575227.201919:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ba00. 10000000:00000010:0.0:1788575227.201922:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9a00. 10000000:01000000:0.0:1788575227.201927:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575227.201931:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575227.201937:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575227.201939:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575227.201944:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575227.201946:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575227.201948:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575227.201951:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575227.201953:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575227.201955:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575227.201958:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575227.201961:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575227.201966:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575229.057758:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575229.057780:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575229.057785:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575229.057789:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575229.057790:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575229.057792:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575229.057795:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575229.057797:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575229.057800:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575229.057820:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575229.057822:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575229.057831:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575229.057836:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575229.057842:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575229.057846:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575229.057851:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575229.057858:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788575229.057866:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575229.057870:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575229.057874:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575229.057882:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692a00. 00010000:00000001:0.0:1788575229.057887:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575229.057890:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788575229.057895:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575229.057898:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168d1a to hash 00000020:00000001:0.0:1788575229.057900:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575229.057902:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575229.057905:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575229.057908:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575229.057912:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575229.057916:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575229.057927:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575229.057938:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575229.057945:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575229.057947:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575229.057953:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575229.057956:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788575229.057967:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575229.057969:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000001:0.0:1788575229.057974:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575229.057978:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575229.057981:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575229.057983:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575229.057993:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575229.058002:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575229.058006:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575229.058013:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575229.058017:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575229.058024:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575229.058027:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575229.058029:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575229.058031:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575229.058034:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575229.058037:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575229.058041:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.058043:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575229.058045:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575229.058049:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575229.058058:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810386432:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575229.058064:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.058067:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575229.058070:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575229.058072:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575229.058082:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575229.058090:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575229.058093:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575229.058098:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575229.058100:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575229.058103:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575229.058106:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575229.058108:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575229.058110:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788575229.058113:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575229.058116:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788575229.058119:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058126:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575229.058130:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058137:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575229.058140:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058146:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575229.058148:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575229.058151:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058157:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575229.058160:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575229.058162:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575229.058165:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575229.058178:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575229.058181:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575229.058184:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058190:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575229.058192:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575229.058193:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575229.058195:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788575229.058197:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575229.058200:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575229.058203:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575229.058206:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575229.058212:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575229.058215:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575229.058221:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575229.058224:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575229.058226:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575229.058228:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575229.058231:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575229.058233:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575229.058235:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575229.058237:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575229.058239:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575229.058242:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168d1a from hash 00010000:00000001:0.0:1788575229.058244:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575229.058246:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575229.058247:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575229.058254:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575229.058256:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575229.058258:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575229.058261:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575229.058262:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575229.058264:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575229.058266:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575229.058270:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575229.058276:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575229.058277:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575229.058279:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575229.058281:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575229.058282:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575229.058283:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575229.058284:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575229.058287:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d1a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575229.058295:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692a00 count: 0 00010000:00000001:0.0:1788575229.058305:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575229.058307:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575229.058310:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575229.058315:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4000 x1875440810386432/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575229.058323:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575229.058327:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575229.058328:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575229.058331:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575229.058333:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788575229.058338:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575229.058341:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575229.058342:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4000. 00000100:00000001:0.0:1788575229.058346:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575229.058347:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575229.058354:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575229.058356:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575229.058360:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575232.897750:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310371871, at 4310371904 00000100:00000010:0.0:1788575232.897768:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00000040:0.0:1788575238.209733:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310377183, at 4310377216 00000100:00000010:0.0:1788575238.209747:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00080000:0.0:1788575239.489772:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575239.489784:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575239.489787:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575239.489791:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575239.489795:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575239.489802:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15706 00000400:00000200:0.0:1788575239.489810:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575239.489816:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575239.489820:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575239.489824:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.489827:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.489831:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.489834:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575239.489839:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575239.489841:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575239.489844:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575239.489848:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575239.489851:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575239.489854:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575239.489857:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.489860:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.489862:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.489864:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575239.489887:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575239.489892:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575239.489896:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575239.489900:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575239.489904:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575239.489907:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.489910:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575239.489913:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575239.489919:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575239.489923:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575239.489927:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575239.489928:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.489932:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575239.489935:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575239.489940:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575239.489948:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575239.489955:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.489965:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810387840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575239.489975:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575239.489989:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.489995:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810387840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575239.490021:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575239.490024:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575239.490027:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575239.490029:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575239.490031:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575239.490034:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575239.490037:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15706 00000400:00000200:0.0:1788575239.490041:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575239.490055:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575239.490058:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575239.490062:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490068:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490082:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490090:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575239.490095:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575239.490099:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575239.490104:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575239.490110:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575239.490115:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575239.490121:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575239.490130:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490135:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490143:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490149:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575239.490155:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575239.490161:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490167:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575239.490170:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575239.490178:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575239.490185:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490191:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575239.490197:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575239.490206:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490213:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575239.490220:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575239.490226:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575239.490233:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575239.490237:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490242:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575239.490253:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575239.490259:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575239.490268:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575239.490277:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.490288:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810387968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575239.490307:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575239.490310:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.490320:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810387968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575239.490353:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575239.490360:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575239.490366:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575239.490370:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575239.490372:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575239.490374:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575239.490386:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15706 00000400:00000200:0.0:1788575239.490394:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575239.490402:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575239.490408:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575239.490415:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490419:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490424:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490429:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575239.490433:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575239.490436:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575239.490440:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575239.490443:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575239.490448:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575239.490454:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575239.490466:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490472:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490478:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490482:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575239.490486:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575239.490489:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490494:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575239.490496:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575239.490500:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575239.490505:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490507:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575239.490512:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575239.490516:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490520:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575239.490527:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575239.490533:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575239.490540:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575239.490543:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490547:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575239.490550:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575239.490554:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575239.490561:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575239.490567:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.490577:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810388096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575239.490595:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575239.490599:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.490610:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810388096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575239.490688:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575239.490696:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575239.490709:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575239.490713:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575239.490717:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575239.490720:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575239.490724:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15706 00000400:00000200:0.0:1788575239.490735:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575239.490743:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575239.490749:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575239.490756:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490762:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490765:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490767:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575239.490771:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575239.490774:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575239.490777:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575239.490782:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575239.490786:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575239.490792:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575239.490801:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490806:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.490811:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490818:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575239.490821:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575239.490826:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575239.490829:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575239.490830:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575239.490834:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575239.490838:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490843:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575239.490848:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575239.490859:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490864:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575239.490871:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575239.490876:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575239.490883:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575239.490886:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.490889:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575239.490894:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575239.490898:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575239.490904:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575239.490912:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.490923:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810388224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575239.490941:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575239.490946:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.490956:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810388224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575239.490974:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575239.490979:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575239.490985:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575239.490989:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575239.490993:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575239.490998:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575239.491004:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15706 00000400:00000200:0.0:1788575239.491013:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575239.491020:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575239.491026:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575239.491034:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.491039:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.491045:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.491055:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575239.491061:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575239.491065:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575239.491068:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575239.491074:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575239.491080:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575239.491087:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575239.491094:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.491099:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575239.491106:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575239.491112:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575239.491117:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575239.491126:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575239.491134:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575239.491141:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575239.491147:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575239.491154:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.491160:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575239.491166:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575239.491174:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.491180:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575239.491186:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575239.491189:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575239.491193:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575239.491197:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.491203:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575239.491207:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575239.491212:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575239.491221:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575239.491231:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.491244:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810388352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575239.491298:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575239.491304:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.491316:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810388352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575239.491353:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575239.491364:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15716) 00000100:00000040:0.0:1788575243.329745:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310382302, at 4310382336 00000100:00000010:0.0:1788575243.329763:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788575244.609767:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575244.609778:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575244.609781:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575244.609785:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575244.609789:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575244.609796:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15711 00000400:00000200:0.0:1788575244.609806:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575244.609812:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575244.609815:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575244.609817:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.609820:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.609823:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.609825:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575244.609829:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575244.609830:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575244.609833:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575244.609841:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575244.609843:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575244.609844:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575244.609847:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.609871:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.609873:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.609875:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575244.609877:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575244.609880:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575244.609882:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575244.609884:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575244.609887:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575244.609889:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.609891:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575244.609893:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575244.609899:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575244.609902:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575244.609905:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575244.609907:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.609910:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575244.609913:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575244.609917:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575244.609925:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575244.609933:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.609942:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810388480/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575244.609952:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575244.609957:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.609963:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810388480/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575244.609992:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575244.609996:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575244.610000:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575244.610001:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575244.610003:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575244.610006:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575244.610015:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15711 00000400:00000200:0.0:1788575244.610019:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575244.610023:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575244.610025:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575244.610029:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610031:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610033:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610034:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575244.610036:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575244.610037:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575244.610039:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575244.610044:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575244.610049:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575244.610054:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575244.610062:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610065:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610071:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610076:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575244.610081:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575244.610088:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610092:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575244.610095:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575244.610102:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575244.610108:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610112:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575244.610118:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575244.610136:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610138:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575244.610141:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575244.610146:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575244.610151:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575244.610158:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610161:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575244.610164:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575244.610166:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575244.610171:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575244.610178:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.610187:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810388608/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575244.610198:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575244.610203:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.610209:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810388608/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575244.610228:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575244.610234:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575244.610242:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575244.610246:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575244.610250:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575244.610256:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575244.610261:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15711 00000400:00000200:0.0:1788575244.610268:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575244.610271:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575244.610273:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575244.610275:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610277:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610279:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610284:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575244.610288:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575244.610289:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575244.610291:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575244.610294:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575244.610299:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575244.610310:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575244.610317:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610323:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610328:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610334:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575244.610340:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575244.610347:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610353:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575244.610358:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575244.610363:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575244.610369:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610374:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575244.610377:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575244.610384:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610388:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575244.610392:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575244.610399:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575244.610406:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575244.610409:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610412:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575244.610415:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575244.610420:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575244.610428:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575244.610435:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.610446:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810388736/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575244.610457:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575244.610461:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.610470:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810388736/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575244.610482:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575244.610492:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575244.610499:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575244.610501:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575244.610502:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575244.610508:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575244.610511:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15711 00000400:00000200:0.0:1788575244.610515:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575244.610519:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575244.610525:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575244.610530:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610533:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610538:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610542:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575244.610547:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575244.610551:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575244.610555:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575244.610560:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575244.610566:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575244.610573:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575244.610582:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610588:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610594:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610600:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575244.610607:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575244.610615:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610619:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575244.610623:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575244.610629:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575244.610663:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610666:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575244.610673:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575244.610675:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610676:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575244.610680:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575244.610683:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575244.610689:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575244.610692:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610695:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575244.610698:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575244.610700:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575244.610705:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575244.610711:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.610719:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810388864/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575244.610728:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575244.610732:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.610739:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810388864/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575244.610750:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575244.610755:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575244.610759:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575244.610763:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575244.610765:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575244.610768:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575244.610771:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15711 00000400:00000200:0.0:1788575244.610777:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575244.610781:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575244.610784:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575244.610797:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610799:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610803:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610810:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575244.610811:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575244.610813:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575244.610814:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575244.610816:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575244.610818:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575244.610823:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575244.610829:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610833:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575244.610837:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610843:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575244.610846:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575244.610852:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575244.610855:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575244.610857:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575244.610861:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575244.610864:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610867:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575244.610869:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575244.610874:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610878:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575244.610885:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575244.610889:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575244.610893:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575244.610896:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.610897:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575244.610899:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575244.610902:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575244.610906:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575244.610911:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.610918:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810388992/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575244.610942:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575244.610946:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.610953:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810388992/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575244.610962:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575244.610971:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15721) 00000100:00000040:0.0:1788575248.449759:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310387422, at 4310387456 00000100:00000010:0.0:1788575248.449779:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00080000:0.0:1788575249.729721:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575249.729738:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575249.729740:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575249.729744:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575249.729749:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575249.729757:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15716 00000400:00000200:0.0:1788575249.729765:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575249.729772:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575249.729775:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575249.729779:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.729782:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.729785:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.729787:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575249.729792:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575249.729794:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575249.729797:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575249.729800:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575249.729803:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575249.729805:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575249.729808:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.729828:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.729831:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.729833:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575249.729837:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575249.729840:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575249.729843:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575249.729846:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575249.729850:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575249.729853:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.729856:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575249.729859:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575249.729865:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575249.729869:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575249.729873:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575249.729875:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.729878:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575249.729881:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575249.729886:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575249.729894:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575249.729901:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.729947:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810389120/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575249.729958:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575249.729963:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.729968:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810389120/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575249.729993:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575249.729996:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575249.730004:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575249.730006:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575249.730008:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575249.730011:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575249.730020:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15716 00000400:00000200:0.0:1788575249.730026:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575249.730031:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575249.730034:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575249.730038:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730044:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730049:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730053:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575249.730058:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575249.730062:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575249.730066:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575249.730070:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575249.730076:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575249.730082:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575249.730092:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730097:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730102:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730107:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575249.730111:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575249.730118:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730124:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575249.730128:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575249.730137:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575249.730141:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730144:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575249.730150:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575249.730156:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730161:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575249.730168:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575249.730173:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575249.730176:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575249.730188:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730192:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575249.730196:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575249.730199:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575249.730206:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575249.730212:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.730221:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810389248/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575249.730233:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575249.730237:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.730246:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810389248/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575249.730262:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575249.730266:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575249.730271:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575249.730274:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575249.730277:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575249.730281:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575249.730285:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15716 00000400:00000200:0.0:1788575249.730335:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575249.730343:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575249.730348:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575249.730351:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730356:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730360:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730362:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575249.730368:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575249.730369:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575249.730373:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575249.730375:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575249.730379:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575249.730388:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575249.730395:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730398:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730402:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730404:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575249.730406:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575249.730409:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730413:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575249.730416:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575249.730418:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575249.730422:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730425:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575249.730428:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575249.730430:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730432:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575249.730437:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575249.730441:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575249.730443:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575249.730445:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730448:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575249.730451:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575249.730454:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575249.730461:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575249.730467:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.730477:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810389376/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575249.730496:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575249.730500:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.730507:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810389376/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575249.730525:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575249.730536:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575249.730545:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575249.730547:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575249.730552:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575249.730555:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575249.730562:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15716 00000400:00000200:0.0:1788575249.730569:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575249.730578:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575249.730583:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575249.730591:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730596:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730600:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730606:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575249.730612:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575249.730615:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575249.730618:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575249.730623:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575249.730628:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575249.730665:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575249.730674:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730680:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730687:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730694:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575249.730700:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575249.730708:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730714:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575249.730720:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575249.730726:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575249.730732:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730739:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575249.730751:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575249.730757:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730762:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575249.730769:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575249.730776:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575249.730783:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575249.730786:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.730792:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575249.730795:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575249.730799:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575249.730808:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575249.730816:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.730826:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810389504/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575249.730844:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575249.730848:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.730858:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810389504/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575249.730889:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575249.730895:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575249.730903:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575249.730923:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575249.730926:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575249.730933:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575249.730941:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15716 00000400:00000200:0.0:1788575249.730953:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575249.730961:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575249.730968:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575249.730976:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.730981:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.730988:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.731001:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575249.731008:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575249.731013:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575249.731018:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575249.731023:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575249.731029:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575249.731036:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575249.731046:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.731052:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575249.731057:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575249.731061:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575249.731067:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575249.731075:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575249.731082:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575249.731087:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575249.731092:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575249.731099:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.731105:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575249.731110:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575249.731117:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.731122:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575249.731128:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575249.731133:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575249.731140:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575249.731144:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.731146:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575249.731150:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575249.731154:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575249.731161:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575249.731167:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.731177:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810389632/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575249.731215:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575249.731220:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.731230:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810389632/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575249.731254:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575249.731263:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15726) 00000100:00080000:0.0:1788575250.562267:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575250.562272:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575250.562274:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575250.562276:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575250.562279:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575250.562282:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15721 00000400:00000200:0.0:1788575250.562288:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575250.562294:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575250.562296:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575250.562298:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562301:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562303:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562304:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575250.562307:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575250.562308:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575250.562310:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575250.562311:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575250.562313:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575250.562314:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575250.562316:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562317:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562319:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562320:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575250.562329:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575250.562333:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562336:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575250.562338:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575250.562341:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575250.562344:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562346:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575250.562349:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575250.562353:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575250.562356:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575250.562359:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575250.562361:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562364:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575250.562365:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575250.562369:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575250.562374:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575250.562378:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.562384:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810389888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575250.562390:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575250.562392:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.562395:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810389888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575250.562412:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575250.562414:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575250.562417:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575250.562418:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575250.562420:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575250.562422:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575250.562423:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15721 00000400:00000200:0.0:1788575250.562426:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575250.562436:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575250.562439:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575250.562441:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562443:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562445:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562451:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575250.562455:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575250.562459:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575250.562460:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575250.562464:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575250.562467:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575250.562472:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575250.562479:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562483:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562486:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562490:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575250.562493:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575250.562498:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562504:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575250.562509:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575250.562517:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575250.562524:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562529:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575250.562536:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575250.562544:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562548:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575250.562555:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575250.562559:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575250.562562:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575250.562565:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562568:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575250.562576:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575250.562580:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575250.562586:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575250.562591:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.562600:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810390016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575250.562613:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575250.562615:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.562625:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810390016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575250.562695:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575250.562699:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575250.562703:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575250.562708:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575250.562711:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575250.562719:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575250.562724:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15721 00000400:00000200:0.0:1788575250.562731:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575250.562736:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575250.562739:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575250.562745:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562746:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562751:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562757:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575250.562759:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575250.562762:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575250.562765:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575250.562768:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575250.562770:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575250.562772:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575250.562794:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562799:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562804:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562806:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575250.562810:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575250.562817:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562819:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575250.562822:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575250.562826:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575250.562831:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562835:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575250.562838:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575250.562842:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562845:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575250.562849:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575250.562851:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575250.562855:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575250.562857:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.562859:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575250.562861:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575250.562864:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575250.562869:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575250.562891:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.562899:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810390144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575250.562908:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575250.562910:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.562913:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810390144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575250.562921:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575250.562927:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575250.562936:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575250.562938:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575250.562941:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575250.562944:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575250.562949:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15721 00000400:00000200:0.0:1788575250.562957:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575250.562961:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575250.562966:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575250.562976:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562981:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.562985:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.562989:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575250.562993:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575250.562996:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575250.562999:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575250.563003:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575250.563006:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575250.563013:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575250.563022:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563027:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.563033:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563040:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575250.563048:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575250.563056:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563063:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575250.563065:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575250.563069:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575250.563074:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563079:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575250.563083:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575250.563092:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563097:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575250.563102:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575250.563108:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575250.563114:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575250.563117:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563120:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575250.563122:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575250.563127:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575250.563137:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575250.563144:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.563153:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810390272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575250.563163:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575250.563168:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.563177:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810390272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575250.563198:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575250.563203:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575250.563209:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575250.563210:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575250.563212:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575250.563213:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575250.563216:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15721 00000400:00000200:0.0:1788575250.563222:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575250.563226:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575250.563229:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575250.563234:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563239:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.563243:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563251:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575250.563254:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575250.563257:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575250.563259:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575250.563262:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575250.563266:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575250.563270:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575250.563276:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563279:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575250.563283:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563286:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575250.563290:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575250.563296:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575250.563300:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575250.563305:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575250.563310:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575250.563313:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563314:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575250.563318:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575250.563323:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563328:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575250.563335:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575250.563341:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575250.563347:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575250.563352:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.563357:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575250.563362:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575250.563367:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575250.563374:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575250.563382:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.563390:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810390400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575250.563421:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575250.563425:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.563432:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810390400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575250.563454:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575250.563460:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15727) 00000100:00000040:0.0:1788575254.401742:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310393375, at 4310393408 00000100:00000010:0.0:1788575254.401757:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000800:00000200:0.0:1788575255.809918:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788575255.873747:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575255.873759:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575255.873762:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575255.873766:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575255.873772:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575255.873781:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15722 00000400:00000200:0.0:1788575255.873792:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575255.873799:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575255.873802:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575255.873806:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.873809:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.873812:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.873814:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575255.873819:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575255.873821:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575255.873824:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575255.873828:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575255.873831:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575255.873833:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575255.873837:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.873855:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.873857:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.873860:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575255.873863:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575255.873867:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575255.873878:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575255.873882:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575255.873886:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575255.873889:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.873892:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575255.873895:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575255.873902:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575255.873906:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575255.873909:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575255.873911:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.873914:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575255.873917:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575255.873922:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575255.873929:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575255.873936:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.873945:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810390528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575255.873953:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575255.873956:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.873960:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810390528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575255.873977:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575255.873980:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575255.873982:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575255.873984:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575255.873985:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575255.873987:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575255.873996:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15722 00000400:00000200:0.0:1788575255.874002:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575255.874007:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575255.874010:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575255.874017:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874023:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874029:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874036:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575255.874043:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575255.874046:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575255.874051:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575255.874055:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575255.874060:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575255.874065:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575255.874072:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874077:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874082:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874086:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575255.874090:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575255.874097:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874102:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575255.874106:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575255.874114:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575255.874119:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874121:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575255.874127:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575255.874135:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874140:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575255.874147:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575255.874151:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575255.874156:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575255.874165:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874171:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575255.874174:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575255.874176:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575255.874181:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575255.874186:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.874192:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810390656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575255.874205:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575255.874209:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.874216:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810390656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575255.874238:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575255.874241:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575255.874249:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575255.874251:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575255.874253:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575255.874258:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575255.874263:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15722 00000400:00000200:0.0:1788575255.874270:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575255.874276:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575255.874278:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575255.874285:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874289:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874294:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874298:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575255.874301:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575255.874304:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575255.874308:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575255.874310:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575255.874321:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575255.874329:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575255.874338:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874341:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874345:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874348:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575255.874351:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575255.874358:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874363:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575255.874365:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575255.874369:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575255.874373:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874378:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575255.874381:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575255.874387:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874393:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575255.874398:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575255.874400:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575255.874405:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575255.874408:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874413:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575255.874415:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575255.874419:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575255.874424:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575255.874429:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.874437:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810390784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575255.874449:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575255.874454:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.874458:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810390784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575255.874471:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575255.874478:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575255.874482:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575255.874485:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575255.874487:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575255.874492:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575255.874496:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15722 00000400:00000200:0.0:1788575255.874503:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575255.874508:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575255.874512:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575255.874517:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874521:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874524:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874528:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575255.874529:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575255.874531:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575255.874534:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575255.874537:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575255.874540:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575255.874542:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575255.874548:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874555:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874559:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874564:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575255.874565:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575255.874570:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874574:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575255.874578:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575255.874581:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575255.874585:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874588:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575255.874594:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575255.874599:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874603:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575255.874608:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575255.874612:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575255.874614:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575255.874617:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874622:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575255.874624:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575255.874627:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575255.874631:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575255.874671:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.874676:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810390912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575255.874684:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575255.874689:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.874697:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810390912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575255.874708:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575255.874712:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575255.874715:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575255.874718:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575255.874720:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575255.874725:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575255.874729:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15722 00000400:00000200:0.0:1788575255.874736:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575255.874742:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575255.874744:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575255.874750:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874755:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874759:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874768:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575255.874772:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575255.874773:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575255.874777:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575255.874780:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575255.874782:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575255.874784:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575255.874790:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874792:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575255.874793:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874795:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575255.874796:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575255.874798:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575255.874799:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575255.874800:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575255.874803:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575255.874808:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874810:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575255.874812:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575255.874815:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874817:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575255.874822:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575255.874827:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575255.874832:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575255.874834:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.874837:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575255.874840:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575255.874842:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575255.874849:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575255.874853:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.874860:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810391040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575255.874900:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575255.874906:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.874917:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810391040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575255.874936:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575255.874944:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15732) 00000100:00000040:0.0:1788575259.713738:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310398686, at 4310398720 00000100:00000010:0.0:1788575259.713753:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000100:00000040:0.0:1788575264.833718:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310403807, at 4310403840 00000100:00000010:0.0:1788575264.833742:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00000040:0.0:1788575269.953734:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310408927, at 4310408960 00000100:00000010:0.0:1788575269.953749:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00000040:0.0:1788575276.353734:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310415327, at 4310415360 00000100:00000010:0.0:1788575276.353749:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000100:00000040:0.0:1788575281.729732:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310420702, at 4310420736 00000100:00000010:0.0:1788575281.729757:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00000040:0.0:1788575286.849719:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310425822, at 4310425856 00000100:00000010:0.0:1788575286.849730:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00000040:0.0:1788575291.969726:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310430942, at 4310430976 00000100:00000010:0.0:1788575291.969741:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 10000000:00000001:0.0:1788575292.737732:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575292.737749:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9d00. 00000020:00000010:0.0:1788575292.737755:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985d600. 00000020:00000010:0.0:1788575292.737761:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985dc00. 00000040:00000010:0.0:1788575292.737765:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575292.737769:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575292.737774:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575292.737777:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575292.737789:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788575292.737792:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788575292.737796:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575292.737798:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575292.737802:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985dc00. 00000040:00000010:0.0:1788575292.737807:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575292.737811:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575292.737813:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575292.737815:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575292.737817:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788575292.737819:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575292.737821:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985d600. 10000000:00000010:0.0:1788575292.737822:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9d00. 10000000:01000000:0.0:1788575292.737824:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575292.737826:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575292.737828:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575292.737829:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575292.737830:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575292.737832:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575292.737833:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575292.737835:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575292.737836:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575292.737838:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575292.737840:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575292.737841:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575292.737842:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575294.465734:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575294.465741:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575294.465746:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575294.465748:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575294.465750:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575294.465751:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575294.465755:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575294.465756:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575294.465759:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575294.465761:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575294.465779:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575294.465787:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575294.465792:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575294.465797:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575294.465801:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575294.465806:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575294.465814:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575294.465820:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575294.465825:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575294.465828:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575294.465835:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788575294.465839:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575294.465843:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30800. 00000020:00000001:0.0:1788575294.465847:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575294.465850:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30800 with handle 0x1441586093168d2f to hash 00000020:00000001:0.0:1788575294.465852:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575294.465853:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575294.465856:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575294.465859:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575294.465863:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575294.465867:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575294.465877:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575294.465888:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575294.465894:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575294.465896:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575294.465902:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575294.465904:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788575294.465907:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575294.465922:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531987712 : -109714177563904 : ffff9c372d6b6300) 00000100:00000001:0.0:1788575294.465926:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575294.465931:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575294.465933:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575294.465935:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575294.465944:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575294.465952:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575294.465963:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575294.465969:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575294.465973:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575294.465980:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575294.465982:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575294.465984:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575294.465987:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575294.465990:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575294.465992:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575294.465995:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575294.465997:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575294.466000:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575294.466004:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575294.466012:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810396416:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575294.466018:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575294.466020:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575294.466022:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575294.466024:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575294.466028:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575294.466045:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575294.466048:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575294.466054:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575294.466056:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575294.466065:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575294.466069:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575294.466070:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575294.466073:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=4 00000020:00000001:0.0:1788575294.466075:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575294.466078:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00010000:0.0:1788575294.466081:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466088:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575294.466092:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466098:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575294.466100:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466106:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575294.466107:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575294.466110:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466122:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575294.466123:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575294.466126:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575294.466129:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575294.466144:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575294.466146:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575294.466148:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466153:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575294.466155:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575294.466156:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575294.466158:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=5 00000020:00000001:0.0:1788575294.466160:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575294.466162:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575294.466165:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575294.466168:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575294.466174:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575294.466176:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575294.466182:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575294.466185:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575294.466186:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575294.466189:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575294.466191:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575294.466194:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575294.466195:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575294.466197:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575294.466198:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575294.466201:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30800 with handle 0x1441586093168d2f from hash 00010000:00000001:0.0:1788575294.466203:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575294.466205:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575294.466206:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575294.466213:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575294.466215:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575294.466217:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575294.466219:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575294.466220:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575294.466221:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575294.466223:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575294.466226:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575294.466232:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575294.466233:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575294.466234:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575294.466236:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575294.466237:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575294.466238:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575294.466239:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575294.466242:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d2f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575294.466248:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788575294.466257:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575294.466258:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575294.466261:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575294.466265:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6300 x1875440810396416/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575294.466272:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575294.466274:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575294.466276:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575294.466278:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575294.466280:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575294.466284:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575294.466286:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575294.466288:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6300. 00000100:00000001:0.0:1788575294.466291:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575294.466292:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575294.466298:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575294.466301:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575294.466304:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000800:00000200:0.0:1788575294.721920:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575298.305718:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310437279, at 4310437312 00000100:00000010:0.0:1788575298.305729:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788575300.801791:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575300.801795:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575300.801799:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575303.745695:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310442718, at 4310442752 00000100:00000010:0.0:1788575303.745709:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788575305.921832:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575305.921834:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575305.921837:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575308.865741:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310447838, at 4310447872 00000100:00000010:0.0:1788575308.865756:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788575310.145731:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575310.145739:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575310.145741:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575310.145745:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575310.145748:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575310.145754:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15776 00000400:00000200:0.0:1788575310.145761:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575310.145766:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575310.145768:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575310.145770:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145773:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.145775:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145777:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575310.145782:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575310.145785:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575310.145805:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575310.145809:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575310.145812:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575310.145815:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575310.145819:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145821:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.145824:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145826:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575310.145829:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575310.145833:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145836:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575310.145851:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575310.145855:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575310.145857:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.145859:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575310.145861:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575310.145867:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575310.145870:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575310.145873:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575310.145874:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.145877:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575310.145879:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575310.145883:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575310.145889:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575310.145894:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.145901:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810398464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575310.145908:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575310.145912:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.145915:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810398464/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575310.145938:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575310.145947:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575310.145951:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575310.145953:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575310.145955:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575310.145958:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575310.145962:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15776 00000400:00000200:0.0:1788575310.145968:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575310.145972:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575310.145974:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575310.145976:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145979:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.145983:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.145986:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575310.145989:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575310.145991:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575310.145994:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575310.145997:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575310.146001:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575310.146004:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575310.146011:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146015:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146017:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146020:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575310.146024:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575310.146029:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146033:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575310.146039:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575310.146046:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575310.146051:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146054:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575310.146064:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575310.146068:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146071:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575310.146076:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575310.146080:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575310.146083:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575310.146086:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146088:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575310.146089:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575310.146092:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575310.146099:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575310.146102:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.146111:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810398592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575310.146140:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575310.146146:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.146155:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810398592/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575310.146169:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575310.146174:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575310.146177:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575310.146179:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575310.146182:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575310.146187:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575310.146192:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15776 00000400:00000200:0.0:1788575310.146226:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575310.146232:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575310.146236:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575310.146240:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146244:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146250:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146260:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575310.146264:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575310.146267:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575310.146271:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575310.146273:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575310.146276:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575310.146279:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575310.146283:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146286:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146291:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146294:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575310.146295:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575310.146297:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146299:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575310.146304:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575310.146307:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575310.146310:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146314:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575310.146317:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575310.146321:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146326:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575310.146331:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575310.146334:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575310.146336:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575310.146339:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146341:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575310.146345:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575310.146347:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575310.146353:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575310.146360:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.146367:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810398720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575310.146392:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575310.146395:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.146399:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810398720/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575310.146411:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575310.146415:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575310.146423:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575310.146426:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575310.146429:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575310.146434:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575310.146441:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15776 00000400:00000200:0.0:1788575310.146449:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575310.146458:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575310.146464:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575310.146473:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146480:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146487:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146494:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575310.146501:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575310.146505:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575310.146510:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575310.146515:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575310.146522:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575310.146529:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575310.146539:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146545:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146549:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146554:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575310.146564:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575310.146569:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146572:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575310.146573:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575310.146576:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575310.146577:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146579:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575310.146583:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575310.146587:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146592:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575310.146598:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575310.146604:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575310.146611:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575310.146615:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146622:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575310.146625:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575310.146631:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575310.146677:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575310.146685:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.146698:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810398848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575310.146717:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575310.146722:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.146733:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810398848/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575310.146757:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575310.146765:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575310.146773:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575310.146777:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575310.146781:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575310.146786:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575310.146798:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15776 00000400:00000200:0.0:1788575310.146807:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575310.146816:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575310.146821:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575310.146829:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146835:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146853:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146859:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575310.146866:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575310.146871:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575310.146876:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575310.146881:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575310.146887:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575310.146890:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575310.146900:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146907:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575310.146913:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146920:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575310.146927:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575310.146936:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575310.146940:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575310.146943:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575310.146947:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575310.146951:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146954:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575310.146957:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575310.146961:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146963:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575310.146968:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575310.146972:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575310.146976:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575310.146983:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575310.146986:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575310.146989:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575310.146993:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575310.147000:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575310.147008:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.147020:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810398976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575310.147049:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575310.147052:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.147059:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810398976/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575310.147083:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575310.147089:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15786) 00000020:00000001:0.0:1788575311.041861:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575311.041865:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575311.041868:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575313.985770:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310452958, at 4310452992 00000100:00000010:0.0:1788575313.985789:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00080000:0.0:1788575315.265758:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575315.265767:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575315.265769:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575315.265772:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575315.265776:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575315.265781:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15781 00000400:00000200:0.0:1788575315.265789:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575315.265794:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575315.265797:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575315.265799:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265802:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.265804:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265829:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575315.265832:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575315.265834:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575315.265836:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575315.265839:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575315.265841:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575315.265842:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575315.265845:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265846:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.265847:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265848:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575315.265851:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575315.265853:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265855:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575315.265857:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575315.265860:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575315.265862:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.265864:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575315.265865:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575315.265872:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575315.265875:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575315.265877:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575315.265879:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.265882:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575315.265884:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575315.265888:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575315.265893:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575315.265899:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.265906:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810399104/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575315.265913:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575315.265917:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.265923:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810399104/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575315.265942:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575315.265945:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575315.265949:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575315.265951:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575315.265952:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575315.265955:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575315.265958:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15781 00000400:00000200:0.0:1788575315.265962:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575315.265966:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575315.265968:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575315.265971:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265974:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.265976:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265978:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575315.265981:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575315.265982:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575315.265984:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575315.265986:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575315.265988:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575315.265990:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575315.265994:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.265996:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.265998:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266000:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575315.266002:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575315.266005:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266008:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575315.266015:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575315.266020:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575315.266022:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266024:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575315.266029:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575315.266032:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266034:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575315.266037:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575315.266040:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575315.266043:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575315.266044:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266046:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575315.266048:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575315.266049:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575315.266053:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575315.266057:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.266062:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810399232/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575315.266069:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575315.266071:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.266074:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810399232/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575315.266084:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575315.266087:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575315.266091:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575315.266093:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575315.266094:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575315.266098:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575315.266100:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15781 00000400:00000200:0.0:1788575315.266104:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575315.266107:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575315.266109:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575315.266115:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266118:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266120:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266121:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575315.266124:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575315.266125:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575315.266127:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575315.266140:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575315.266145:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575315.266148:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575315.266151:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266153:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266154:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266157:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575315.266160:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575315.266162:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266165:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575315.266167:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575315.266171:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575315.266173:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266175:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575315.266177:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575315.266180:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266181:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575315.266184:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575315.266187:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575315.266189:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575315.266190:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266192:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575315.266194:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575315.266196:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575315.266200:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575315.266209:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.266214:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810399360/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575315.266232:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575315.266234:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.266238:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810399360/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575315.266247:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575315.266249:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575315.266254:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575315.266255:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575315.266257:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575315.266260:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575315.266262:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15781 00000400:00000200:0.0:1788575315.266267:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575315.266269:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575315.266272:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575315.266274:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266276:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266279:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266281:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575315.266283:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575315.266284:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575315.266286:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575315.266288:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575315.266290:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575315.266293:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575315.266296:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266298:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266304:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266306:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575315.266309:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575315.266312:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266314:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575315.266316:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575315.266319:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575315.266321:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266324:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575315.266327:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575315.266329:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266330:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575315.266334:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575315.266336:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575315.266339:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575315.266340:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266342:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575315.266344:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575315.266346:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575315.266350:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575315.266354:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.266358:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810399488/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575315.266367:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575315.266369:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.266372:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810399488/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575315.266382:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575315.266384:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575315.266388:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575315.266390:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575315.266395:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575315.266398:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575315.266400:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15781 00000400:00000200:0.0:1788575315.266404:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575315.266407:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575315.266410:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575315.266413:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266415:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266417:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266419:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575315.266421:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575315.266423:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575315.266425:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575315.266427:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575315.266439:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575315.266441:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575315.266445:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266447:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575315.266449:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266451:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575315.266454:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575315.266457:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575315.266459:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575315.266461:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575315.266464:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575315.266467:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266469:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575315.266471:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575315.266473:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266476:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575315.266483:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575315.266485:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575315.266487:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575315.266489:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.266490:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575315.266492:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575315.266494:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575315.266498:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575315.266501:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.266506:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810399616/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575315.266514:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575315.266516:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.266520:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810399616/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575315.266529:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575315.266533:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15791) 00000020:00000001:0.0:1788575316.161855:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575316.161860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575316.161864:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575316.674240:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575316.674247:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575316.674250:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575316.674252:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575316.674255:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575316.674259:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15786 00000400:00000200:0.0:1788575316.674265:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575316.674269:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575316.674272:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575316.674274:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674276:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674278:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674286:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575316.674289:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575316.674290:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575316.674291:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575316.674293:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575316.674295:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575316.674297:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575316.674299:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674301:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674302:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674303:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575316.674305:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575316.674314:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674316:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575316.674318:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575316.674320:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575316.674322:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674324:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575316.674325:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575316.674328:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575316.674330:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575316.674332:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575316.674334:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674335:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575316.674336:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575316.674339:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575316.674342:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575316.674345:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.674350:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810399872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575316.674356:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575316.674358:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.674365:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810399872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575316.674384:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575316.674386:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575316.674389:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575316.674390:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575316.674391:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575316.674393:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575316.674394:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15786 00000400:00000200:0.0:1788575316.674397:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575316.674399:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575316.674400:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575316.674402:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674404:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674405:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674407:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575316.674408:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575316.674409:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575316.674410:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575316.674412:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575316.674414:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575316.674416:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575316.674418:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674419:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674421:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674423:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575316.674425:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575316.674427:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674429:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575316.674438:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575316.674445:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575316.674449:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674454:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575316.674460:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575316.674464:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674468:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575316.674472:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575316.674476:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575316.674481:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575316.674483:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674487:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575316.674490:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575316.674493:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575316.674498:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575316.674504:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.674512:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810400000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575316.674523:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575316.674526:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.674531:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810400000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575316.674542:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575316.674545:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575316.674549:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575316.674552:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575316.674560:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575316.674567:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575316.674572:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15786 00000400:00000200:0.0:1788575316.674577:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575316.674584:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575316.674588:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575316.674598:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674601:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674603:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674605:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575316.674609:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575316.674612:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575316.674615:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575316.674616:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575316.674617:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575316.674619:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575316.674625:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674629:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674691:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674696:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575316.674700:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575316.674706:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674711:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575316.674716:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575316.674721:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575316.674724:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674726:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575316.674731:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575316.674734:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674740:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575316.674743:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575316.674748:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575316.674753:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575316.674756:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674757:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575316.674759:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575316.674762:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575316.674767:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575316.674777:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.674785:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810400128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575316.674796:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575316.674800:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.674806:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810400128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575316.674818:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575316.674822:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575316.674828:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575316.674830:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575316.674833:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575316.674837:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575316.674841:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15786 00000400:00000200:0.0:1788575316.674848:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575316.674854:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575316.674859:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575316.674864:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674868:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674872:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674876:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575316.674878:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575316.674882:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575316.674884:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575316.674887:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575316.674891:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575316.674894:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575316.674897:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674899:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.674907:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674910:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575316.674914:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575316.674920:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575316.674924:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575316.674927:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575316.674931:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575316.674936:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674941:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575316.674944:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575316.674947:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674951:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575316.674955:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575316.674960:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575316.674964:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575316.674967:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.674970:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575316.674973:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575316.674975:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575316.674979:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575316.674985:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.674997:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810400256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575316.675030:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575316.675034:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.675046:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810400256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575316.675070:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575316.675077:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575316.675088:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575316.675095:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575316.675104:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575316.675112:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575316.675118:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15786 00000400:00000200:0.0:1788575316.675124:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575316.675132:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575316.675138:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575316.675145:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.675150:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.675157:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.675162:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575316.675170:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575316.675174:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575316.675179:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575316.675183:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575316.675189:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575316.675197:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575316.675204:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.675209:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575316.675215:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575316.675221:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575316.675225:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575316.675232:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575316.675238:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575316.675240:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575316.675242:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575316.675244:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.675248:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575316.675250:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575316.675253:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.675258:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575316.675271:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575316.675279:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575316.675286:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575316.675291:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.675297:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575316.675302:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575316.675307:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575316.675314:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575316.675321:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.675330:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810400384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575316.675344:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575316.675349:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.675356:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810400384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575316.675381:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575316.675388:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15793) 00000100:00000040:0.0:1788575320.513740:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310459487, at 4310459520 00000100:00000010:0.0:1788575320.513760:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788575321.281800:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575321.281803:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575321.281806:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575325.761777:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310464734, at 4310464768 00000100:00000010:0.0:1788575325.761794:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788575326.401789:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575326.401793:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575326.401797:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575327.041777:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575327.041786:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575327.041788:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575327.041791:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575327.041794:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575327.041816:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15793 00000400:00000200:0.0:1788575327.041823:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575327.041828:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575327.041831:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575327.041833:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041836:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.041838:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041839:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575327.041842:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575327.041844:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575327.041846:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575327.041849:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575327.041850:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575327.041852:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575327.041853:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041855:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.041856:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041857:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575327.041860:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575327.041863:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041864:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575327.041867:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575327.041869:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575327.041871:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.041873:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575327.041875:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575327.041879:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575327.041882:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575327.041885:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575327.041887:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.041894:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575327.041896:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575327.041903:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575327.041909:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575327.041916:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.041923:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810401152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575327.041930:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575327.041934:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.041938:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810401152/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575327.041955:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575327.041958:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575327.041960:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575327.041961:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575327.041963:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575327.041965:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575327.041966:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15793 00000400:00000200:0.0:1788575327.041969:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575327.041971:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575327.041972:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575327.041974:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041976:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.041977:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041978:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575327.041980:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575327.041981:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575327.041982:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575327.041984:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575327.041986:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575327.041987:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575327.041989:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.041996:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.041998:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042000:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575327.042005:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575327.042007:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042012:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575327.042015:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575327.042019:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575327.042024:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042029:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575327.042035:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575327.042043:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042048:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575327.042055:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575327.042062:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575327.042068:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575327.042072:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042077:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575327.042081:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575327.042086:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575327.042095:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575327.042103:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.042113:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810401280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575327.042130:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575327.042133:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.042139:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810401280/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575327.042170:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575327.042175:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575327.042181:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575327.042190:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575327.042192:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575327.042196:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575327.042199:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15793 00000400:00000200:0.0:1788575327.042209:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575327.042216:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575327.042221:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575327.042227:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042232:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042238:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042242:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575327.042249:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575327.042251:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575327.042253:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575327.042256:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575327.042261:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575327.042265:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575327.042276:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042279:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042283:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042286:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575327.042288:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575327.042294:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042299:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575327.042316:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575327.042320:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575327.042322:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042325:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575327.042327:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575327.042331:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042339:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575327.042343:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575327.042346:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575327.042357:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575327.042361:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042362:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575327.042363:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575327.042365:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575327.042369:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575327.042372:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.042378:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810401408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575327.042390:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575327.042394:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.042398:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810401408/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575327.042406:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575327.042410:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575327.042414:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575327.042416:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575327.042418:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575327.042422:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575327.042425:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15793 00000400:00000200:0.0:1788575327.042429:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575327.042435:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575327.042441:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575327.042444:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042449:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042452:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042454:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575327.042459:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575327.042467:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575327.042471:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575327.042476:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575327.042479:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575327.042484:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575327.042492:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042498:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042503:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042507:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575327.042512:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575327.042521:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042526:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575327.042530:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575327.042533:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575327.042536:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042541:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575327.042544:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575327.042547:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042549:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575327.042555:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575327.042561:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575327.042565:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575327.042567:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042571:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575327.042575:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575327.042579:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575327.042584:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575327.042590:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.042597:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810401536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575327.042607:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575327.042610:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.042621:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810401536/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575327.042632:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575327.042662:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575327.042667:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575327.042669:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575327.042670:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575327.042677:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575327.042681:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15793 00000400:00000200:0.0:1788575327.042687:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575327.042691:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575327.042693:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575327.042698:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042700:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042703:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042707:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575327.042711:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575327.042713:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575327.042716:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575327.042718:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575327.042721:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575327.042725:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575327.042731:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042733:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575327.042739:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042743:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575327.042749:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575327.042754:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575327.042758:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575327.042768:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575327.042774:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575327.042779:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042784:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575327.042790:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575327.042795:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042800:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575327.042806:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575327.042813:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575327.042819:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575327.042823:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.042828:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575327.042832:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575327.042837:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575327.042844:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575327.042852:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.042861:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810401664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575327.042877:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575327.042881:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.042891:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810401664/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575327.042914:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575327.042936:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15803) 00000100:00000040:0.0:1788575330.881717:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310469854, at 4310469888 00000100:00000010:0.0:1788575330.881738:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788575331.521820:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575331.521825:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575331.521829:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575332.609927:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575336.001866:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310474974, at 4310475008 00000100:00000010:0.0:1788575336.001891:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788575336.641890:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575336.641908:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575336.641911:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575338.689750:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575338.689757:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575338.689764:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575338.689767:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575338.689769:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575338.689770:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575338.689771:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575338.689773:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575338.689775:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575338.689778:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575338.689779:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575338.689786:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575338.689791:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575338.689796:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575338.689799:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575338.689804:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575338.689814:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575338.689821:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575338.689824:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575338.689827:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575338.689834:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788575338.689837:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575338.689839:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788575338.689842:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575338.689844:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168d3d to hash 00000020:00000001:0.0:1788575338.689846:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575338.689847:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575338.689849:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575338.689852:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575338.689855:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575338.689858:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575338.689866:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575338.689881:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575338.689885:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575338.689887:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575338.689891:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575338.689893:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185609472 : -109714523942144 : ffff9c3718c61300) 00000100:00000001:0.0:1788575338.689895:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575338.689896:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531978752 : -109714177572864 : ffff9c372d6b4000) 00000100:00000001:0.0:1788575338.689900:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575338.689903:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575338.689905:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575338.689906:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575338.689915:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575338.689922:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575338.689925:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575338.689930:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575338.689933:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575338.689937:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575338.689939:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575338.689940:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575338.689942:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575338.689943:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575338.689945:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575338.689948:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575338.689949:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575338.689951:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575338.689957:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575338.689964:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810403072:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575338.689967:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575338.689969:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575338.689971:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575338.689972:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575338.689975:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575338.689980:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575338.689982:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61300. 00000100:00000001:0.0:1788575338.689986:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575338.689987:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575338.689989:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575338.689992:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575338.689993:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575338.689995:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788575338.689997:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575338.689999:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788575338.690001:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690005:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575338.690018:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690024:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575338.690026:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690037:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575338.690038:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575338.690040:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690043:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575338.690045:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575338.690046:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575338.690049:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575338.690053:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575338.690054:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575338.690056:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690060:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575338.690061:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575338.690062:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575338.690063:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788575338.690065:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575338.690066:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575338.690068:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575338.690070:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575338.690085:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575338.690087:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575338.690091:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575338.690096:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575338.690097:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575338.690100:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575338.690101:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575338.690104:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575338.690105:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575338.690107:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575338.690108:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575338.690110:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168d3d from hash 00010000:00000001:0.0:1788575338.690111:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575338.690112:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575338.690113:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575338.690114:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575338.690115:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575338.690116:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575338.690117:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575338.690118:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575338.690119:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575338.690120:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575338.690122:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575338.690125:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575338.690126:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575338.690127:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575338.690128:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575338.690129:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575338.690130:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575338.690131:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575338.690133:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168d3d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575338.690138:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788575338.690144:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575338.690146:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575338.690147:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575338.690154:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4000 x1875440810403072/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575338.690159:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575338.690162:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575338.690163:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575338.690165:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575338.690167:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575338.690170:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575338.690172:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575338.690173:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4000. 00000100:00000001:0.0:1788575338.690176:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575338.690176:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575338.690178:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575338.690180:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575338.690183:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575341.761805:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575341.761808:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575341.761811:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575342.529910:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310481503, at 4310481536 00000100:00000010:0.0:1788575342.529915:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788575346.881836:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575346.881841:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575346.881845:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575347.777739:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310486750, at 4310486784 00000100:00000010:0.0:1788575347.777755:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00080000:0.0:1788575349.057728:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.057736:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575349.057739:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.057742:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.057745:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.057751:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15815 00000400:00000200:0.0:1788575349.057759:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575349.057764:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.057781:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.057784:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057786:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.057788:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057789:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575349.057793:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.057794:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575349.057796:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.057799:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575349.057800:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575349.057802:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575349.057804:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057806:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.057807:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057808:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575349.057811:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575349.057813:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057814:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.057817:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.057819:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.057821:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.057823:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575349.057825:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.057829:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575349.057832:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575349.057835:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.057836:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.057838:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.057840:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.057843:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.057849:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575349.057855:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.057865:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810404480/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.057873:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.057876:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.057880:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810404480/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.057896:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.057899:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.057901:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575349.057903:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.057904:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.057906:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.057907:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15815 00000400:00000200:0.0:1788575349.057910:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575349.057912:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.057913:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.057915:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057932:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.057933:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057935:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575349.057936:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.057937:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575349.057938:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.057940:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575349.057941:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575349.057943:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575349.057945:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057947:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.057948:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057954:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575349.057956:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575349.057958:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575349.057961:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.057962:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.057967:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.057970:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.057975:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.057978:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575349.057981:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.057984:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.057987:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575349.057991:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575349.057995:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.057998:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058002:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.058004:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.058008:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.058013:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575349.058018:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.058026:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810404608/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.058038:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.058042:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.058049:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810404608/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.058063:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.058068:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.058070:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575349.058071:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.058073:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.058081:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.058085:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15815 00000400:00000200:0.0:1788575349.058091:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575349.058096:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.058098:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.058104:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058105:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058108:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058112:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575349.058115:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.058116:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575349.058117:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.058119:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575349.058124:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575349.058128:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575349.058134:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058136:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058140:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058142:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575349.058147:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575349.058151:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058157:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.058162:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.058166:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.058169:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058173:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.058177:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575349.058181:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058185:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.058188:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575349.058195:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575349.058199:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.058202:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058204:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.058205:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.058206:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.058210:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575349.058215:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.058221:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810404736/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.058234:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.058237:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.058242:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810404736/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.058252:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.058255:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.058260:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575349.058263:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.058266:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.058269:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.058274:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15815 00000400:00000200:0.0:1788575349.058319:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575349.058324:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.058327:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.058329:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058331:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058334:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058337:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575349.058339:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.058340:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575349.058346:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.058351:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575349.058353:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575349.058360:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575349.058366:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058371:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058373:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058377:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575349.058381:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575349.058386:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058392:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.058393:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.058397:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.058403:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058407:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.058409:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575349.058413:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058417:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.058421:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575349.058424:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575349.058429:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.058430:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058434:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.058436:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.058437:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.058441:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575349.058445:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.058451:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810404864/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.058478:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.058481:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.058485:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810404864/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.058497:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.058499:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.058507:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575349.058509:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.058513:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.058517:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.058521:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15815 00000400:00000200:0.0:1788575349.058527:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575349.058531:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.058534:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.058536:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058538:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058543:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058546:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575349.058548:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.058551:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575349.058553:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.058556:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575349.058558:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575349.058560:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575349.058563:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058566:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.058568:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058569:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575349.058571:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575349.058576:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575349.058578:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.058583:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.058585:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.058587:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058592:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.058594:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575349.058596:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058600:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.058603:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575349.058607:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575349.058609:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.058613:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.058615:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.058619:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.058621:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.058627:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575349.058633:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.058684:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810404992/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.058696:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.058701:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.058710:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810404992/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.058730:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.058737:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15825) 00000020:00000001:0.0:1788575352.001999:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575352.002004:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575352.002008:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575352.897746:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310491870, at 4310491904 00000100:00000010:0.0:1788575352.897769:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00080000:0.0:1788575354.177758:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575354.177766:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575354.177768:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575354.177786:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575354.177790:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575354.177797:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15820 00000400:00000200:0.0:1788575354.177805:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575354.177811:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575354.177814:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575354.177818:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.177821:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.177824:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.177827:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575354.177832:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575354.177834:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575354.177837:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575354.177852:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575354.177855:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575354.177858:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575354.177862:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.177864:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.177867:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.177868:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575354.177872:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575354.177875:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575354.177878:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575354.177881:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575354.177885:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575354.177888:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.177890:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575354.177893:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575354.177900:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575354.177904:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575354.177908:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575354.177916:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.177919:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575354.177922:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575354.177925:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575354.177931:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575354.177937:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.177944:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810405120/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575354.177952:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575354.177955:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.177958:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810405120/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575354.177979:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575354.177982:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575354.177984:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575354.177985:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575354.177987:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575354.177989:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575354.177992:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15820 00000400:00000200:0.0:1788575354.177996:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575354.177999:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575354.178001:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575354.178003:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178005:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178007:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178010:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575354.178013:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575354.178014:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575354.178017:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575354.178020:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575354.178023:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575354.178030:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575354.178037:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178042:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178046:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178052:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575354.178058:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575354.178065:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178070:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575354.178076:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575354.178085:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575354.178092:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178098:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575354.178105:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575354.178110:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178115:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575354.178120:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575354.178127:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575354.178132:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575354.178136:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178140:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575354.178143:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575354.178146:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575354.178151:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575354.178157:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.178162:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810405248/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575354.178178:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575354.178183:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.178189:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810405248/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575354.178204:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575354.178216:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575354.178222:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575354.178226:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575354.178230:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575354.178241:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575354.178248:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15820 00000400:00000200:0.0:1788575354.178299:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575354.178309:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575354.178315:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575354.178321:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178327:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178331:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178337:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575354.178341:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575354.178344:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575354.178350:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575354.178353:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575354.178356:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575354.178362:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575354.178370:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178375:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178381:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178387:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575354.178397:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575354.178407:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178412:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575354.178415:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575354.178421:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575354.178426:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178431:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575354.178442:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575354.178447:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178452:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575354.178458:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575354.178464:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575354.178469:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575354.178472:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178475:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575354.178478:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575354.178482:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575354.178488:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575354.178495:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.178504:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810405376/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575354.178516:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575354.178521:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.178529:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810405376/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575354.178546:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575354.178551:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575354.178559:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575354.178562:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575354.178566:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575354.178573:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575354.178579:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15820 00000400:00000200:0.0:1788575354.178589:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575354.178597:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575354.178603:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575354.178612:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178617:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178624:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178670:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575354.178677:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575354.178681:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575354.178687:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575354.178693:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575354.178699:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575354.178706:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575354.178711:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178714:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.178718:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178724:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575354.178732:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575354.178740:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575354.178746:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575354.178752:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575354.178757:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575354.178763:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178768:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575354.178773:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575354.178779:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178783:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575354.178793:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575354.178799:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575354.178806:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575354.178810:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.178812:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575354.178816:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575354.178822:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575354.178831:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575354.178857:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.178869:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810405504/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575354.178909:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575354.178915:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.178927:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810405504/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575354.178953:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575354.178960:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575354.178970:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575354.178975:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575354.178979:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575354.178986:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575354.178994:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15820 00000400:00000200:0.0:1788575354.179009:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575354.179018:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575354.179023:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575354.179032:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.179039:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.179047:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.179054:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575354.179061:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575354.179064:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575354.179070:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575354.179075:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575354.179082:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575354.179087:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575354.179098:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.179104:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575354.179111:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575354.179117:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575354.179129:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575354.179136:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575354.179144:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575354.179148:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575354.179155:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575354.179162:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.179168:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575354.179173:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575354.179181:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.179187:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575354.179196:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575354.179202:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575354.179209:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575354.179213:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575354.179218:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575354.179223:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575354.179228:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575354.179238:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575354.179246:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.179258:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810405632/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575354.179281:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575354.179287:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.179297:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810405632/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575354.179320:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575354.179328:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15830) 00000020:00000001:0.0:1788575357.121854:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575357.121866:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575357.121869:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575358.017741:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310496990, at 4310497024 00000100:00000010:0.0:1788575358.017759:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788575362.241797:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575362.241803:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575362.241808:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575363.841785:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310502815, at 4310502848 00000100:00000010:0.0:1788575363.841801:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000020:00000001:0.0:1788575367.361855:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575367.361860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575367.361863:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575369.281753:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310508254, at 4310508288 00000100:00000010:0.0:1788575369.281775:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000800:00000200:0.0:1788575370.498002:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575372.481784:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575372.481788:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575372.481791:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575374.401782:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310513374, at 4310513408 00000100:00000010:0.0:1788575374.401797:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000020:00000001:0.0:1788575377.601903:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575377.601909:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575377.601912:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575379.521724:0:267927:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310518494, at 4310518528 00000100:00000010:0.0:1788575379.521739:0:267927:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00080000:0.0:1788575380.801755:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575380.801766:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575380.801769:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575380.801773:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575380.801777:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575380.801785:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15847 00000400:00000200:0.0:1788575380.801794:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575380.801800:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575380.801804:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575380.801808:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.801827:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.801830:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.801832:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575380.801835:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575380.801837:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575380.801839:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575380.801843:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575380.801845:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575380.801848:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575380.801851:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.801853:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.801855:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.801857:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575380.801860:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575380.801864:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575380.801867:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.801871:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575380.801874:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.801877:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.801879:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575380.801882:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575380.801888:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575380.801892:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575380.801896:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575380.801898:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.801901:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575380.801905:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575380.801909:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575380.801916:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575380.801923:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.801932:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810408576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575380.801956:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575380.801961:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.801966:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810408576/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.801981:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575380.801984:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575380.801986:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575380.801987:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575380.801989:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575380.801991:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575380.801993:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15847 00000400:00000200:0.0:1788575380.801997:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575380.802001:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575380.802003:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575380.802005:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802007:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802009:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802011:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575380.802015:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575380.802016:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575380.802017:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575380.802019:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575380.802020:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575380.802022:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575380.802025:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802029:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802033:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802035:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575380.802044:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575380.802051:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802055:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.802059:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575380.802064:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.802071:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802078:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575380.802084:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575380.802089:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802096:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575380.802103:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575380.802109:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575380.802115:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575380.802119:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802121:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575380.802122:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575380.802125:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575380.802135:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575380.802144:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.802155:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810408704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575380.802172:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575380.802176:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.802185:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810408704/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.802206:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575380.802212:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575380.802221:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575380.802227:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575380.802229:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575380.802235:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575380.802251:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15847 00000400:00000200:0.0:1788575380.802288:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575380.802296:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575380.802300:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575380.802306:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802311:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802317:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802320:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575380.802327:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575380.802331:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575380.802336:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575380.802342:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575380.802347:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575380.802352:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575380.802360:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802365:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802370:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802374:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575380.802381:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575380.802389:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802395:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.802400:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575380.802407:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.802414:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802419:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575380.802424:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575380.802430:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802465:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575380.802473:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575380.802486:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575380.802493:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575380.802501:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802507:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575380.802511:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575380.802516:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575380.802525:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575380.802533:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.802547:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810408832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575380.802572:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575380.802577:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.802589:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810408832/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.802610:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575380.802615:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575380.802624:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575380.802625:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575380.802627:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575380.802633:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575380.802661:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15847 00000400:00000200:0.0:1788575380.802667:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575380.802672:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575380.802679:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575380.802688:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802694:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802699:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802705:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575380.802709:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575380.802713:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575380.802718:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575380.802722:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575380.802729:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575380.802739:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575380.802745:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802748:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802752:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802756:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575380.802758:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575380.802760:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802764:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.802767:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575380.802772:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.802777:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802781:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575380.802784:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575380.802789:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802792:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575380.802798:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575380.802802:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575380.802804:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575380.802807:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.802811:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575380.802814:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575380.802817:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575380.802823:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575380.802828:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.802836:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810408960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575380.802847:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575380.802851:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.802857:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810408960/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.802872:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575380.802881:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575380.802885:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575380.802888:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575380.802892:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575380.802894:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575380.802898:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15847 00000400:00000200:0.0:1788575380.802902:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575380.802911:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575380.802918:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575380.802929:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802937:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.802945:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.802952:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575380.802960:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575380.802965:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575380.802970:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575380.802976:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575380.802984:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575380.802990:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575380.802999:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.803003:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575380.803008:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575380.803014:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575380.803020:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575380.803029:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575380.803034:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.803039:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575380.803044:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.803049:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.803054:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575380.803063:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575380.803068:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.803071:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575380.803076:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575380.803079:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575380.803084:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575380.803087:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.803090:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575380.803093:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575380.803096:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575380.803102:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575380.803107:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.803114:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810409088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575380.803127:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575380.803132:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.803137:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810409088/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.803154:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.803159:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15857) 00000100:00080000:0.0:1788575382.082484:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575382.082495:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575382.082498:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575382.082501:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575382.082505:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575382.082511:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15852 00000400:00000200:0.0:1788575382.082519:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575382.082525:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575382.082528:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575382.082530:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082533:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.082557:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082559:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575382.082562:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575382.082564:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575382.082566:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575382.082569:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575382.082571:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575382.082572:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575382.082574:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082575:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.082577:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082578:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575382.082580:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575382.082582:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082584:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575382.082586:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575382.082589:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575382.082591:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.082593:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575382.082594:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575382.082598:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575382.082601:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575382.082604:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575382.082605:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.082608:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575382.082610:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575382.082614:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575382.082619:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575382.082624:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.082631:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810409344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575382.082704:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575382.082715:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.082723:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810409344/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575382.082751:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575382.082755:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575382.082759:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575382.082761:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575382.082763:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575382.082766:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575382.082769:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15852 00000400:00000200:0.0:1788575382.082774:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575382.082778:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575382.082781:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575382.082785:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082788:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.082791:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082797:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575382.082802:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575382.082806:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575382.082809:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575382.082815:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575382.082822:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575382.082829:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575382.082838:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082843:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.082850:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082855:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575382.082861:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575382.082870:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575382.082887:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575382.082891:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575382.082898:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575382.082904:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.082907:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575382.082913:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575382.082917:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.082922:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575382.082927:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575382.082933:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575382.082937:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575382.082940:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.082943:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575382.082945:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575382.082946:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575382.082953:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575382.082960:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.082968:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810409472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575382.082983:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575382.082987:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.082996:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810409472/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575382.083006:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575382.083010:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575382.083013:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575382.083016:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575382.083018:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575382.083021:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575382.083025:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15852 00000400:00000200:0.0:1788575382.083039:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575382.083051:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575382.083055:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575382.083061:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083066:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083069:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083072:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575382.083076:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575382.083080:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575382.083085:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575382.083089:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575382.083096:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575382.083108:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575382.083115:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083119:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083125:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083131:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575382.083137:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575382.083144:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083152:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575382.083157:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575382.083161:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575382.083166:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083172:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575382.083176:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575382.083183:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083187:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575382.083193:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575382.083200:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575382.083208:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575382.083212:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083217:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575382.083227:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575382.083232:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575382.083241:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575382.083250:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.083260:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810409600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575382.083275:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575382.083279:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.083289:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810409600/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575382.083308:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575382.083314:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575382.083323:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575382.083327:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575382.083331:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575382.083337:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575382.083344:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15852 00000400:00000200:0.0:1788575382.083354:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575382.083362:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575382.083367:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575382.083377:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083382:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083385:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083387:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575382.083389:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575382.083392:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575382.083397:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575382.083402:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575382.083409:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575382.083434:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575382.083449:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083455:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083461:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083467:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575382.083473:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575382.083481:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083486:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575382.083492:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575382.083498:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575382.083506:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083512:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575382.083516:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575382.083523:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083528:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575382.083537:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575382.083541:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575382.083544:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575382.083547:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083550:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575382.083554:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575382.083559:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575382.083563:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575382.083565:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.083573:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810409728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575382.083580:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575382.083584:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.083591:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810409728/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575382.083610:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575382.083614:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575382.083625:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575382.083628:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575382.083631:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575382.083659:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575382.083663:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15852 00000400:00000200:0.0:1788575382.083668:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575382.083670:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575382.083672:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575382.083675:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083678:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083681:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083686:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575382.083689:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575382.083693:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575382.083694:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575382.083698:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575382.083702:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575382.083708:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575382.083712:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083716:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575382.083723:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083726:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575382.083728:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575382.083733:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575382.083738:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575382.083742:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575382.083746:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575382.083751:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083756:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575382.083759:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575382.083769:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083773:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575382.083777:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575382.083783:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575382.083789:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575382.083792:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575382.083796:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575382.083799:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575382.083802:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575382.083808:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575382.083814:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.083824:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810409856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575382.083839:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575382.083842:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.083849:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810409856/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575382.083867:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575382.083872:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15858) 00000020:00000001:0.0:1788575382.721815:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575382.721818:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575382.721822:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575385.921753:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310524895, at 4310524928 00000100:00000010:0.0:1788575385.921768:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00080000:0.0:1788575387.457748:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575387.457758:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575387.457760:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575387.457763:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575387.457766:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575387.457772:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15853 00000400:00000200:0.0:1788575387.457780:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575387.457800:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575387.457810:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575387.457813:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457816:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.457818:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457820:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575387.457824:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575387.457826:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575387.457829:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575387.457833:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575387.457835:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575387.457837:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575387.457839:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457841:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.457843:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457844:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575387.457847:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575387.457850:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457853:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575387.457855:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575387.457859:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575387.457861:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.457863:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575387.457864:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575387.457870:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575387.457873:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575387.457876:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575387.457877:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.457879:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575387.457882:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575387.457886:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575387.457895:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575387.457909:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.457918:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810409984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575387.457926:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575387.457930:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.457935:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810409984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575387.457958:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575387.457961:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575387.457964:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575387.457966:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575387.457967:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575387.457969:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575387.457973:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15853 00000400:00000200:0.0:1788575387.457979:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575387.457983:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575387.457986:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575387.457989:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457992:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.457994:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.457998:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575387.458000:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575387.458003:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575387.458006:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575387.458010:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575387.458014:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575387.458021:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575387.458032:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458038:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458058:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458068:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575387.458077:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575387.458091:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458099:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575387.458105:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575387.458114:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575387.458122:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458125:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575387.458130:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575387.458135:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458138:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575387.458148:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575387.458158:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575387.458168:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575387.458170:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458175:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575387.458178:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575387.458181:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575387.458192:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575387.458203:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.458224:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810410112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575387.458246:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575387.458251:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.458261:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810410112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575387.458284:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575387.458288:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575387.458295:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575387.458306:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575387.458319:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575387.458328:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575387.458350:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15853 00000400:00000200:0.0:1788575387.458355:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575387.458365:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575387.458373:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575387.458384:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458390:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458401:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458407:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575387.458417:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575387.458420:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575387.458425:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575387.458430:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575387.458433:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575387.458435:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575387.458438:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458440:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458441:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458443:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575387.458444:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575387.458446:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458448:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575387.458449:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575387.458452:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575387.458453:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458455:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575387.458456:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575387.458458:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458459:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575387.458475:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575387.458477:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575387.458479:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575387.458480:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458482:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575387.458483:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575387.458485:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575387.458488:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575387.458491:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.458495:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810410240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575387.458500:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575387.458502:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.458504:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810410240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575387.458511:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575387.458512:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575387.458515:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575387.458516:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575387.458517:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575387.458519:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575387.458520:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15853 00000400:00000200:0.0:1788575387.458524:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575387.458526:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575387.458527:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575387.458529:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458531:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458532:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458534:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575387.458535:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575387.458539:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575387.458540:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575387.458542:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575387.458543:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575387.458545:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575387.458547:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458548:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458549:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458550:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575387.458552:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575387.458553:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458554:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575387.458555:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575387.458557:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575387.458559:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458560:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575387.458562:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575387.458563:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458564:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575387.458566:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575387.458567:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575387.458569:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575387.458570:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458571:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575387.458572:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575387.458573:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575387.458577:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575387.458583:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.458586:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810410368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575387.458598:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575387.458602:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.458614:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810410368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575387.458654:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575387.458658:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575387.458660:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575387.458662:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575387.458663:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575387.458666:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575387.458669:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15853 00000400:00000200:0.0:1788575387.458673:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575387.458677:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575387.458679:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575387.458683:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458686:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458690:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458694:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575387.458697:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575387.458698:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575387.458699:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575387.458701:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575387.458706:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575387.458708:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575387.458713:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458719:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575387.458722:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458726:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575387.458728:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575387.458731:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575387.458734:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575387.458742:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575387.458746:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575387.458751:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458754:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575387.458757:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575387.458760:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458762:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575387.458765:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575387.458769:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575387.458775:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575387.458777:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575387.458781:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575387.458784:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575387.458787:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575387.458793:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575387.458798:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.458805:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810410496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575387.458823:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575387.458828:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.458838:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810410496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575387.458853:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575387.458859:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15864) 00000020:00000001:0.0:1788575387.841757:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575387.841761:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575387.841764:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575391.297782:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310530271, at 4310530304 00000100:00000010:0.0:1788575391.297809:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215d20. 00000100:00080000:0.0:1788575392.577715:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575392.577722:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575392.577724:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575392.577744:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575392.577747:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575392.577753:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15859 00000400:00000200:0.0:1788575392.577760:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575392.577765:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575392.577767:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575392.577770:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577772:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.577774:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577776:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575392.577779:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575392.577780:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575392.577782:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575392.577785:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575392.577786:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575392.577788:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575392.577790:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577791:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.577793:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577794:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575392.577796:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575392.577799:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577801:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575392.577803:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575392.577806:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575392.577808:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.577809:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575392.577811:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575392.577815:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575392.577818:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575392.577821:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575392.577825:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.577827:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575392.577830:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575392.577834:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575392.577840:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575392.577845:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.577852:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810410624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575392.577859:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575392.577863:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.577866:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810410624/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575392.577886:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575392.577888:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575392.577891:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575392.577892:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575392.577893:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575392.577895:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575392.577897:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15859 00000400:00000200:0.0:1788575392.577900:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575392.577902:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575392.577903:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575392.577905:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577907:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.577909:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577911:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575392.577913:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575392.577914:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575392.577916:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575392.577917:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575392.577919:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575392.577924:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575392.577930:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577935:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.577940:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577943:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575392.577948:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575392.577953:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575392.577960:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575392.577963:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575392.577972:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575392.577981:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.577988:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575392.577994:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575392.577998:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578004:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575392.578010:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575392.578014:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575392.578019:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575392.578021:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578026:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575392.578032:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575392.578038:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575392.578046:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575392.578054:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.578062:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810410752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575392.578072:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575392.578075:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.578080:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810410752/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575392.578093:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575392.578103:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575392.578109:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575392.578113:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575392.578115:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575392.578120:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575392.578124:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15859 00000400:00000200:0.0:1788575392.578131:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575392.578136:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575392.578138:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575392.578143:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578145:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578146:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578149:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575392.578152:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575392.578154:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575392.578156:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575392.578158:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575392.578160:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575392.578165:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575392.578168:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578172:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578177:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578179:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575392.578181:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575392.578186:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578189:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575392.578192:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575392.578196:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575392.578202:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578204:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575392.578210:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575392.578215:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578219:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575392.578222:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575392.578224:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575392.578229:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575392.578231:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578236:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575392.578237:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575392.578241:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575392.578248:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575392.578253:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.578285:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810410880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575392.578292:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575392.578294:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.578299:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810410880/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575392.578317:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575392.578321:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575392.578326:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575392.578328:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575392.578330:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575392.578335:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575392.578338:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15859 00000400:00000200:0.0:1788575392.578344:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575392.578349:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575392.578353:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575392.578360:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578364:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578367:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578377:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575392.578381:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575392.578384:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575392.578387:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575392.578392:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575392.578397:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575392.578402:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575392.578409:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578415:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578419:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578425:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575392.578430:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575392.578435:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578438:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575392.578439:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575392.578442:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575392.578447:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578453:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575392.578456:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575392.578464:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578469:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575392.578478:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575392.578485:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575392.578490:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575392.578494:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578497:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575392.578498:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575392.578500:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575392.578508:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575392.578513:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.578524:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810411008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575392.578571:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575392.578574:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.578580:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810411008/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575392.578594:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575392.578600:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575392.578605:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575392.578607:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575392.578609:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575392.578615:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575392.578619:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15859 00000400:00000200:0.0:1788575392.578626:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575392.578632:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575392.578655:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575392.578657:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578659:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578660:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578661:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575392.578663:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575392.578664:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575392.578666:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575392.578667:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575392.578669:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575392.578671:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575392.578676:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578681:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575392.578683:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578685:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575392.578693:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575392.578700:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575392.578706:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575392.578711:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575392.578715:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575392.578720:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578722:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575392.578725:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575392.578727:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578730:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575392.578737:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575392.578743:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575392.578750:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575392.578754:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575392.578759:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575392.578762:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575392.578768:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575392.578776:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575392.578783:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.578794:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810411136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575392.578809:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575392.578814:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.578823:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810411136/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575392.578842:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575392.578848:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15869) 00000020:00000001:0.0:1788575392.961848:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575392.961853:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575392.961856:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575396.417775:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310535390, at 4310535424 00000100:00000010:0.0:1788575396.417794:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215900. 00000100:00080000:0.0:1788575397.697721:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575397.697736:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575397.697739:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575397.697744:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575397.697751:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575397.697760:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15864 00000400:00000200:0.0:1788575397.697772:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575397.697779:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575397.697783:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575397.697787:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.697792:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.697795:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.697798:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575397.697803:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575397.697806:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575397.697809:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575397.697813:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575397.697816:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575397.697819:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575397.697822:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.697825:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.697827:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.697829:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575397.697833:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575397.697837:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575397.697840:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575397.697843:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575397.697847:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575397.697851:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.697853:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575397.697875:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575397.697883:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575397.697888:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575397.697893:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575397.697895:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.697899:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575397.697908:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575397.697914:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575397.697924:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575397.697934:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.697946:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810411264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575397.697972:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575397.697978:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.697984:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810411264/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575397.698021:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575397.698026:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575397.698030:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575397.698032:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575397.698034:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575397.698037:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575397.698040:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15864 00000400:00000200:0.0:1788575397.698045:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575397.698048:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575397.698051:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575397.698055:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698057:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698059:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698062:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575397.698065:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575397.698072:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575397.698074:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575397.698077:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575397.698079:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575397.698082:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575397.698086:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698088:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698090:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698092:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575397.698100:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575397.698103:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698106:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575397.698108:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575397.698116:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575397.698119:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698121:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575397.698127:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575397.698130:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698133:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575397.698137:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575397.698140:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575397.698143:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575397.698145:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698147:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575397.698149:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575397.698151:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575397.698156:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575397.698160:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.698165:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810411392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575397.698173:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575397.698175:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.698185:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810411392/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575397.698194:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575397.698197:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575397.698213:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575397.698216:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575397.698218:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575397.698221:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575397.698223:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15864 00000400:00000200:0.0:1788575397.698228:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575397.698231:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575397.698234:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575397.698242:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698244:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698247:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698249:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575397.698252:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575397.698253:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575397.698255:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575397.698257:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575397.698260:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575397.698262:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575397.698266:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698268:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698270:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698272:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575397.698274:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575397.698277:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698279:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575397.698286:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575397.698289:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575397.698291:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698293:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575397.698296:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575397.698299:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698301:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575397.698305:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575397.698308:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575397.698310:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575397.698312:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698314:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575397.698316:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575397.698318:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575397.698322:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575397.698326:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.698332:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810411520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575397.698340:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575397.698342:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.698347:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810411520/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575397.698356:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575397.698359:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575397.698363:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575397.698364:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575397.698366:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575397.698369:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575397.698371:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15864 00000400:00000200:0.0:1788575397.698375:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575397.698378:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575397.698380:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575397.698388:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698390:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698392:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698395:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575397.698397:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575397.698399:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575397.698400:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575397.698402:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575397.698405:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575397.698407:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575397.698410:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698412:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698414:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698417:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575397.698419:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575397.698423:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698425:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575397.698426:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575397.698429:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575397.698431:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698433:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575397.698436:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575397.698439:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698441:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575397.698444:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575397.698447:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575397.698450:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575397.698451:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698453:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575397.698455:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575397.698457:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575397.698461:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575397.698468:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.698473:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810411648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575397.698482:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575397.698484:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.698488:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810411648/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575397.698496:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575397.698499:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575397.698504:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575397.698505:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575397.698513:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575397.698516:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575397.698519:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15864 00000400:00000200:0.0:1788575397.698523:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575397.698527:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575397.698529:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575397.698533:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698535:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698538:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698540:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575397.698542:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575397.698543:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575397.698545:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575397.698547:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575397.698550:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575397.698552:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575397.698556:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698557:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575397.698564:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698566:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575397.698570:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575397.698573:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575397.698576:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575397.698578:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575397.698580:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575397.698582:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698585:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575397.698587:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575397.698589:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698591:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575397.698597:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575397.698600:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575397.698603:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575397.698604:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.698606:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575397.698608:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575397.698610:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575397.698614:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575397.698618:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.698623:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810411776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575397.698630:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575397.698632:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.698664:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810411776/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575397.698673:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575397.698679:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15874) 00000020:00000001:0.0:1788575398.081917:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575398.081922:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575398.081927:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575401.537744:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310540511, at 4310540544 00000100:00000010:0.0:1788575401.537754:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322156c0. 10000000:00000001:0.0:1788575402.305759:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575402.305776:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a90c0. 00000020:00000010:0.0:1788575402.305785:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575402.305791:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575402.305795:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788575402.305801:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575402.305809:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575402.305813:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575402.305817:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575402.305821:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985d600. 10000000:00000040:0.0:1788575402.305826:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575402.305829:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575402.305834:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575402.305841:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788575402.305843:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575402.305846:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575402.305848:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575402.305851:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575402.305853:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985d600. 00000020:00000010:0.0:1788575402.305856:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ba00. 10000000:00000010:0.0:1788575402.305857:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a90c0. 10000000:01000000:0.0:1788575402.305860:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575402.305862:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575402.305864:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575402.305866:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575402.305867:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575402.305869:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575402.305871:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575402.305874:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575402.305875:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575402.305876:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575402.305894:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575402.305895:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575402.305897:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788575403.201961:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575403.201963:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575403.201966:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575403.521726:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575403.521732:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575403.521735:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575403.521736:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575403.521737:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575403.521739:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575403.521741:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575403.521742:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575403.521744:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575403.521746:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575403.521747:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575403.521755:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575403.521759:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575403.521763:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575403.521766:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575403.521770:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575403.521778:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985d600. 00000100:00000001:0.0:1788575403.521784:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575403.521788:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575403.521790:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575403.521800:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692700. 00010000:00000001:0.0:1788575403.521804:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575403.521807:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30a00. 00000020:00000001:0.0:1788575403.521810:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575403.521813:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30a00 with handle 0x1441586093168d52 to hash 00000020:00000001:0.0:1788575403.521815:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575403.521817:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575403.521819:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575403.521822:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575403.521826:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575403.521842:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575403.521852:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575403.521862:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575403.521867:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575403.521870:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575403.521876:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575403.521878:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575403.521881:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575403.521882:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531981440 : -109714177570176 : ffff9c372d6b4a80) 00000100:00000001:0.0:1788575403.521887:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575403.521891:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575403.521892:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575403.521894:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575403.521903:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575403.521911:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575403.521914:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575403.521919:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575403.521923:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575403.521928:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575403.521931:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575403.521933:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575403.521935:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575403.521937:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575403.521944:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575403.521948:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.521950:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575403.521952:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575403.521957:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575403.521964:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810412544:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575403.521968:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.521969:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575403.521971:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575403.521973:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575403.521975:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575403.521986:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575403.521988:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575403.521992:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575403.521993:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575403.521995:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575403.521997:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575403.521999:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575403.522000:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=4 00000020:00000001:0.0:1788575403.522002:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575403.522004:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00010000:0.0:1788575403.522006:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522010:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575403.522013:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522023:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575403.522025:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522028:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575403.522029:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575403.522030:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522033:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575403.522035:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575403.522036:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575403.522038:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575403.522043:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575403.522045:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575403.522047:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522052:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575403.522053:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575403.522054:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575403.522055:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788575403.522057:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575403.522059:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575403.522062:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575403.522065:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575403.522071:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575403.522078:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575403.522085:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575403.522087:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575403.522089:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575403.522090:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575403.522092:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575403.522094:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575403.522095:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575403.522102:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575403.522103:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575403.522105:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30a00 with handle 0x1441586093168d52 from hash 00010000:00000001:0.0:1788575403.522107:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575403.522108:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575403.522109:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575403.522109:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575403.522110:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575403.522111:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575403.522113:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575403.522113:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575403.522114:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575403.522116:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575403.522117:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575403.522121:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575403.522121:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575403.522122:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575403.522123:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575403.522124:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575403.522124:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575403.522125:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575403.522126:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168d52 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575403.522130:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692700 count: 0 00010000:00000001:0.0:1788575403.522168:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575403.522170:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575403.522171:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575403.522175:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4a80 x1875440810412544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575403.522179:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575403.522181:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575403.522182:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575403.522183:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575403.522185:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985d600. 02000000:00000001:0.0:1788575403.522188:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575403.522189:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575403.522190:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4a80. 00000100:00000001:0.0:1788575403.522192:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575403.522193:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575403.522194:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575403.522196:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575403.522198:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575407.361751:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310546335, at 4310546368 00000100:00000010:0.0:1788575407.361767:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000020:00000001:0.0:1788575408.321807:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575408.321812:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575408.321815:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575408.961788:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575408.961799:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575408.961801:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575408.961805:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575408.961809:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575408.961815:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15875 00000400:00000200:0.0:1788575408.961824:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575408.961830:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575408.961834:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575408.961837:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.961852:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.961855:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.961858:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575408.961861:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575408.961863:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575408.961864:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575408.961867:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575408.961868:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575408.961870:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575408.961873:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.961874:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.961876:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.961878:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575408.961881:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575408.961884:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575408.961887:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575408.961890:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575408.961893:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575408.961897:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.961899:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575408.961902:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575408.961909:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575408.961912:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575408.961917:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575408.961919:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.961922:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575408.961925:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575408.961929:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575408.961937:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575408.961944:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.961953:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810413312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575408.961974:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575408.961979:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.961984:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810413312/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575408.962008:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575408.962012:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575408.962016:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575408.962018:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575408.962021:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575408.962024:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575408.962028:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15875 00000400:00000200:0.0:1788575408.962033:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575408.962038:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575408.962041:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575408.962044:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962049:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962054:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962059:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575408.962063:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575408.962065:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575408.962069:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575408.962073:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575408.962077:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575408.962081:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575408.962088:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962092:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962110:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962115:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575408.962127:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575408.962134:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962139:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575408.962144:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575408.962152:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575408.962156:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962160:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575408.962165:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575408.962170:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962173:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575408.962178:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575408.962182:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575408.962184:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575408.962185:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962187:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575408.962189:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575408.962191:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575408.962195:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575408.962200:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.962207:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810413440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575408.962220:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575408.962224:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.962234:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810413440/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575408.962250:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575408.962254:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575408.962259:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575408.962262:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575408.962265:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575408.962268:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575408.962278:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15875 00000400:00000200:0.0:1788575408.962282:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575408.962284:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575408.962286:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575408.962288:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962289:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962290:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962292:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575408.962293:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575408.962294:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575408.962295:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575408.962296:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575408.962297:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575408.962299:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575408.962300:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962302:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962303:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962304:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575408.962305:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575408.962306:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962307:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575408.962308:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575408.962311:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575408.962312:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962313:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575408.962315:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575408.962316:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962317:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575408.962319:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575408.962320:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575408.962322:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575408.962325:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962326:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575408.962328:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575408.962329:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575408.962332:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575408.962334:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.962340:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810413568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575408.962348:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575408.962350:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.962355:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810413568/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575408.962363:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575408.962367:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575408.962371:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575408.962372:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575408.962374:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575408.962376:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575408.962378:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15875 00000400:00000200:0.0:1788575408.962380:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575408.962382:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575408.962383:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575408.962384:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962385:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962386:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962387:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575408.962389:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575408.962389:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575408.962390:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575408.962391:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575408.962393:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575408.962397:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575408.962399:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962401:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962402:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962405:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575408.962407:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575408.962409:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962411:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575408.962413:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575408.962415:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575408.962417:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962419:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575408.962421:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575408.962424:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962425:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575408.962429:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575408.962431:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575408.962434:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575408.962435:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962437:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575408.962438:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575408.962440:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575408.962444:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575408.962447:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.962452:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810413696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575408.962459:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575408.962461:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.962465:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810413696/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575408.962471:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575408.962480:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575408.962486:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575408.962488:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575408.962490:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575408.962493:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575408.962497:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15875 00000400:00000200:0.0:1788575408.962542:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575408.962549:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575408.962552:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575408.962558:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962560:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962563:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962566:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575408.962569:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575408.962571:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575408.962574:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575408.962576:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575408.962578:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575408.962580:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575408.962587:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962591:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575408.962595:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962597:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575408.962602:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575408.962608:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575408.962613:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575408.962615:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575408.962618:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575408.962620:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962622:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575408.962658:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575408.962660:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962662:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575408.962665:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575408.962667:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575408.962669:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575408.962671:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.962672:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575408.962673:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575408.962682:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575408.962686:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575408.962690:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.962695:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810413824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575408.962701:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575408.962703:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.962707:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810413824/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575408.962714:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575408.962719:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15885) 00000800:00000200:0.0:1788575409.409935:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575412.801735:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310551775, at 4310551808 00000100:00000010:0.0:1788575412.801753:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788575413.441979:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575413.441986:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575413.441991:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575417.921696:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310556894, at 4310556928 00000100:00000010:0.0:1788575417.921713:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 00000020:00000001:0.0:1788575418.561827:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575418.561830:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575418.561834:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575423.041776:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310562014, at 4310562048 00000100:00000010:0.0:1788575423.041795:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000020:00000001:0.0:1788575423.681812:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575423.681816:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575423.681818:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575423.809721:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575423.809735:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a90c0. 00000020:00000010:0.0:1788575423.809741:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985d600. 00000020:00000010:0.0:1788575423.809746:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575423.809748:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985dc00. 00000020:00000010:0.0:1788575423.809752:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575423.809757:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575423.809760:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575423.809764:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575423.809767:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788575423.809770:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575423.809772:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575423.809776:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575423.809782:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985dc00. 00000020:00000010:0.0:1788575423.809785:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575423.809787:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575423.809790:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575423.809792:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575423.809794:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575423.809795:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985d600. 10000000:00000010:0.0:1788575423.809797:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a90c0. 10000000:01000000:0.0:1788575423.809799:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575423.809802:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575423.809803:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575423.809805:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575423.809806:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575423.809808:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575423.809809:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575423.809811:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575423.809812:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575423.809813:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575423.809826:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575423.809827:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575423.809829:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788575424.321753:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575424.321762:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575424.321763:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575424.321766:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575424.321769:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575424.321775:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15890 00000400:00000200:0.0:1788575424.321783:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575424.321788:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575424.321791:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575424.321794:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321797:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.321799:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321802:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575424.321805:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575424.321807:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575424.321809:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575424.321812:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575424.321814:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575424.321816:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575424.321819:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321820:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.321822:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321823:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575424.321826:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575424.321829:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321831:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575424.321834:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575424.321837:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575424.321856:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.321859:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575424.321861:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575424.321867:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575424.321870:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575424.321874:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575424.321875:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.321878:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575424.321881:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575424.321885:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575424.321892:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575424.321898:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.321906:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810415232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575424.321914:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575424.321918:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.321922:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810415232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575424.321949:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575424.321962:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575424.321965:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575424.321967:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575424.321969:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575424.321971:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575424.321974:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15890 00000400:00000200:0.0:1788575424.321979:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575424.321983:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575424.321985:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575424.321988:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.321993:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.321996:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322003:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575424.322006:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575424.322008:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575424.322009:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575424.322011:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575424.322012:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575424.322014:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575424.322018:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322020:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322022:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322026:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575424.322030:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575424.322034:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322040:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575424.322044:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575424.322050:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575424.322053:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322057:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575424.322060:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575424.322063:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322066:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575424.322070:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575424.322074:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575424.322077:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575424.322079:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322082:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575424.322084:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575424.322089:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575424.322096:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575424.322101:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.322110:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810415360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575424.322137:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575424.322140:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.322145:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810415360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575424.322158:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575424.322163:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575424.322171:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575424.322178:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575424.322179:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575424.322182:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575424.322184:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15890 00000400:00000200:0.0:1788575424.322190:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575424.322194:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575424.322196:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575424.322202:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322204:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322206:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322211:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575424.322214:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575424.322219:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575424.322222:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575424.322227:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575424.322233:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575424.322239:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575424.322248:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322254:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322258:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322264:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575424.322276:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575424.322284:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322291:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575424.322294:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575424.322298:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575424.322302:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322307:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575424.322312:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575424.322319:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322324:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575424.322330:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575424.322336:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575424.322342:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575424.322346:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322348:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575424.322352:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575424.322354:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575424.322360:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575424.322364:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.322372:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810415488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575424.322380:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575424.322383:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.322388:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810415488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575424.322398:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575424.322401:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575424.322403:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575424.322404:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575424.322405:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575424.322407:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575424.322417:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15890 00000400:00000200:0.0:1788575424.322425:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575424.322434:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575424.322438:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575424.322446:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322450:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322454:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322460:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575424.322466:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575424.322470:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575424.322474:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575424.322478:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575424.322484:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575424.322490:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575424.322507:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322516:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322521:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322527:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575424.322532:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575424.322539:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322545:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575424.322551:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575424.322558:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575424.322564:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322569:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575424.322574:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575424.322580:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322584:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575424.322592:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575424.322597:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575424.322603:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575424.322610:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322615:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575424.322616:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575424.322620:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575424.322625:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575424.322631:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.322674:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810415616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575424.322683:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575424.322688:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.322697:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810415616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575424.322710:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575424.322714:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575424.322720:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575424.322723:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575424.322727:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575424.322732:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575424.322736:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15890 00000400:00000200:0.0:1788575424.322741:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575424.322747:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575424.322751:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575424.322754:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322757:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322761:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322764:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575424.322767:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575424.322768:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575424.322770:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575424.322772:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575424.322776:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575424.322785:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575424.322790:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322794:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575424.322798:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322801:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575424.322803:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575424.322806:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575424.322807:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575424.322810:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575424.322813:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575424.322818:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322820:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575424.322824:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575424.322830:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322833:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575424.322839:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575424.322843:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575424.322848:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575424.322850:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.322853:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575424.322856:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575424.322858:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575424.322863:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575424.322867:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.322875:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810415744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575424.322886:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575424.322889:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.322896:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810415744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575424.322907:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575424.322917:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15901) 10000000:00000001:0.0:1788575425.793738:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575425.793744:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575425.793748:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575425.793751:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575425.793752:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575425.793753:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575425.793756:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575425.793757:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575425.793759:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575425.793761:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575425.793762:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575425.793771:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575425.793776:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575425.793781:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575425.793784:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575425.793788:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575425.793796:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788575425.793802:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575425.793806:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575425.793809:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575425.793817:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692700. 00010000:00000001:0.0:1788575425.793820:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575425.793823:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30800. 00000020:00000001:0.0:1788575425.793827:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575425.793829:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30800 with handle 0x1441586093168d59 to hash 00000020:00000001:0.0:1788575425.793831:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575425.793832:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575425.793834:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575425.793836:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575425.793839:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575425.793841:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575425.793849:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575425.793869:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575425.793873:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575425.793875:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575425.793880:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575425.793881:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575425.793883:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575425.793884:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531990400 : -109714177561216 : ffff9c372d6b6d80) 00000100:00000001:0.0:1788575425.793887:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575425.793891:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575425.793893:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575425.793895:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575425.793906:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575425.793915:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575425.793919:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575425.793926:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575425.793930:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575425.793936:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575425.793972:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575425.793974:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575425.793977:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575425.793980:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575425.793982:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575425.793986:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.793988:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575425.793990:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575425.794001:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575425.794006:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810415872:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575425.794011:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.794013:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575425.794015:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575425.794016:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575425.794018:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575425.794022:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575425.794024:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575425.794028:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575425.794029:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575425.794030:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575425.794033:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575425.794034:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575425.794036:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=4 00000020:00000001:0.0:1788575425.794037:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575425.794039:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00010000:0.0:1788575425.794041:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794045:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575425.794049:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794053:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575425.794054:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794063:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575425.794064:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575425.794066:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794069:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575425.794070:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575425.794072:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575425.794074:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575425.794077:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575425.794078:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575425.794080:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794084:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575425.794085:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575425.794086:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575425.794087:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=5 00000020:00000001:0.0:1788575425.794088:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575425.794089:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575425.794090:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575425.794092:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575425.794095:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575425.794097:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575425.794099:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575425.794103:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575425.794104:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575425.794106:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575425.794107:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575425.794109:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575425.794111:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575425.794112:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575425.794113:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575425.794114:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30800 with handle 0x1441586093168d59 from hash 00010000:00000001:0.0:1788575425.794115:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575425.794116:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575425.794117:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575425.794117:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575425.794118:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575425.794119:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575425.794121:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575425.794122:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575425.794122:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575425.794124:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575425.794126:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575425.794128:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575425.794129:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575425.794130:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575425.794131:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575425.794132:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575425.794132:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575425.794133:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575425.794134:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168d59 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575425.794138:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692700 count: 0 00010000:00000001:0.0:1788575425.794144:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575425.794145:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575425.794146:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575425.794151:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6d80 x1875440810415872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575425.794156:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575425.794158:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575425.794159:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575425.794161:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575425.794163:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788575425.794170:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575425.794172:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575425.794173:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6d80. 00000100:00000001:0.0:1788575425.794176:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575425.794177:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575425.794178:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575425.794180:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575425.794182:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575428.801795:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575428.801799:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575428.801801:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575429.633778:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310568607, at 4310568640 00000100:00000010:0.0:1788575429.633796:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788575433.921808:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575433.921812:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575433.921815:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575434.817774:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310573790, at 4310573824 00000100:00000010:0.0:1788575434.817793:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000020:00000001:0.0:1788575439.041836:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575439.041840:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575439.041844:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575439.937726:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310578910, at 4310578944 00000100:00000010:0.0:1788575439.937739:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000100:00080000:0.0:1788575441.217777:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575441.217790:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575441.217792:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575441.217828:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575441.217832:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575441.217837:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15907 00000400:00000200:0.0:1788575441.217844:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575441.217848:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575441.217850:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575441.217853:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217855:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.217856:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217858:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575441.217861:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575441.217862:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575441.217864:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575441.217866:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575441.217868:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575441.217869:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575441.217871:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217872:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.217873:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217874:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575441.217876:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575441.217879:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217880:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575441.217882:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575441.217884:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575441.217886:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.217887:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575441.217889:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575441.217893:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575441.217895:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575441.217898:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575441.217902:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.217904:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575441.217906:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575441.217909:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575441.217914:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575441.217920:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.217926:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810417920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575441.217935:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575441.217939:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.217943:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810417920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575441.217972:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575441.217976:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575441.217978:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575441.217979:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575441.217980:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575441.217983:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575441.217984:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15907 00000400:00000200:0.0:1788575441.217987:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575441.217989:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575441.217991:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575441.217993:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217994:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.217995:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.217997:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575441.217998:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575441.217999:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575441.218000:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575441.218002:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575441.218004:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575441.218011:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575441.218014:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218017:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218020:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218024:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575441.218030:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575441.218037:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218043:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575441.218047:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575441.218054:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575441.218059:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218062:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575441.218067:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575441.218071:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218076:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575441.218080:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575441.218084:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575441.218087:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575441.218091:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218092:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575441.218093:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575441.218097:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575441.218101:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575441.218105:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.218111:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810418048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575441.218118:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575441.218120:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.218125:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810418048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575441.218137:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575441.218146:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575441.218150:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575441.218152:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575441.218154:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575441.218156:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575441.218160:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15907 00000400:00000200:0.0:1788575441.218164:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575441.218170:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575441.218171:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575441.218176:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218180:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218183:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218188:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575441.218191:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575441.218194:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575441.218197:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575441.218202:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575441.218206:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575441.218211:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575441.218218:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218222:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218227:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218229:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575441.218234:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575441.218239:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218243:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575441.218247:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575441.218249:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575441.218255:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218259:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575441.218267:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575441.218270:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218271:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575441.218278:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575441.218283:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575441.218287:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575441.218290:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218295:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575441.218298:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575441.218300:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575441.218304:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575441.218310:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.218318:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810418176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575441.218328:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575441.218331:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.218335:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810418176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575441.218345:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575441.218350:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575441.218354:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575441.218357:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575441.218358:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575441.218362:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575441.218367:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15907 00000400:00000200:0.0:1788575441.218372:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575441.218378:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575441.218381:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575441.218385:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218388:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218389:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218396:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575441.218399:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575441.218402:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575441.218404:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575441.218406:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575441.218409:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575441.218410:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575441.218413:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218414:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218418:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218419:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575441.218424:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575441.218427:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218431:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575441.218432:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575441.218436:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575441.218439:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218443:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575441.218445:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575441.218449:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218451:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575441.218454:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575441.218455:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575441.218457:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575441.218458:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218459:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575441.218460:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575441.218461:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575441.218466:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575441.218469:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.218483:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810418304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575441.218502:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575441.218504:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.218509:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810418304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575441.218520:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575441.218525:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575441.218529:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575441.218533:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575441.218544:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575441.218550:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575441.218554:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15907 00000400:00000200:0.0:1788575441.218560:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575441.218566:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575441.218571:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575441.218576:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218580:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218586:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218591:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575441.218595:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575441.218598:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575441.218602:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575441.218606:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575441.218610:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575441.218615:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575441.218621:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218625:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575441.218628:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218631:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575441.218665:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575441.218671:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575441.218674:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575441.218677:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575441.218681:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575441.218685:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218688:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575441.218690:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575441.218694:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218695:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575441.218701:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575441.218705:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575441.218708:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575441.218710:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.218715:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575441.218716:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575441.218720:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575441.218724:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575441.218729:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.218736:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810418432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575441.218748:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575441.218751:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.218756:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810418432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575441.218766:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575441.218770:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15917) 00000020:00000001:0.0:1788575444.161893:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575444.161899:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575444.161902:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575445.057720:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310584030, at 4310584064 00000100:00000010:0.0:1788575445.057737:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000100:00080000:0.0:1788575446.337744:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575446.337767:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575446.337770:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575446.337776:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575446.337782:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575446.337797:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15912 00000400:00000200:0.0:1788575446.337809:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575446.337817:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575446.337821:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575446.337824:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.337828:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.337831:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.337834:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575446.337838:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575446.337840:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575446.337843:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575446.337847:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575446.337848:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575446.337850:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575446.337852:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.337853:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.337854:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.337855:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575446.337858:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575446.337860:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575446.337862:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575446.337865:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575446.337868:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575446.337870:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.337872:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575446.337896:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575446.337903:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575446.337906:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575446.337910:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575446.337912:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.337915:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575446.337918:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575446.337922:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575446.337930:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575446.337937:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.337945:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810418560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575446.337953:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575446.337957:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.337966:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810418560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575446.337984:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575446.337987:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575446.337989:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575446.337990:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575446.337991:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575446.337993:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575446.337995:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15912 00000400:00000200:0.0:1788575446.337997:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575446.337999:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575446.338000:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575446.338002:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338004:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338005:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338006:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575446.338008:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575446.338013:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575446.338015:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575446.338017:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575446.338018:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575446.338020:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575446.338023:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338025:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338027:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338030:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575446.338033:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575446.338038:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338043:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575446.338047:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575446.338054:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575446.338059:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338064:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575446.338070:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575446.338075:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338078:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575446.338081:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575446.338083:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575446.338085:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575446.338087:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338089:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575446.338091:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575446.338094:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575446.338099:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575446.338104:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.338110:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810418688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575446.338119:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575446.338121:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.338130:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810418688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575446.338140:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575446.338142:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575446.338147:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575446.338148:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575446.338149:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575446.338151:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575446.338153:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15912 00000400:00000200:0.0:1788575446.338158:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575446.338161:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575446.338162:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575446.338166:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338169:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338171:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338173:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575446.338175:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575446.338176:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575446.338179:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575446.338181:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575446.338183:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575446.338184:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575446.338186:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338188:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338191:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338193:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575446.338194:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575446.338196:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338197:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575446.338202:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575446.338205:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575446.338208:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338211:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575446.338213:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575446.338216:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338218:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575446.338222:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575446.338225:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575446.338228:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575446.338230:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338233:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575446.338235:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575446.338237:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575446.338242:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575446.338245:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.338250:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810418816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575446.338263:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575446.338267:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.338273:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810418816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575446.338285:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575446.338290:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575446.338294:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575446.338295:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575446.338296:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575446.338300:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575446.338304:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15912 00000400:00000200:0.0:1788575446.338308:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575446.338312:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575446.338315:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575446.338324:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338327:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338332:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338334:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575446.338336:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575446.338339:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575446.338340:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575446.338343:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575446.338347:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575446.338351:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575446.338355:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338355:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338356:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338358:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575446.338361:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575446.338363:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338364:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575446.338366:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575446.338369:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575446.338374:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338378:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575446.338383:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575446.338388:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338390:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575446.338394:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575446.338398:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575446.338401:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575446.338405:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338406:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575446.338408:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575446.338411:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575446.338417:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575446.338429:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.338437:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810418944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575446.338451:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575446.338456:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.338462:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810418944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575446.338474:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575446.338480:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575446.338485:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575446.338487:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575446.338491:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575446.338497:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575446.338501:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15912 00000400:00000200:0.0:1788575446.338508:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575446.338514:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575446.338518:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575446.338524:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338528:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338532:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338537:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575446.338542:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575446.338544:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575446.338547:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575446.338550:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575446.338553:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575446.338558:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575446.338563:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338564:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575446.338574:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338579:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575446.338583:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575446.338586:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575446.338591:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575446.338592:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575446.338594:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575446.338597:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338600:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575446.338601:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575446.338603:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338604:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575446.338608:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575446.338611:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575446.338612:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575446.338614:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575446.338616:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575446.338618:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575446.338620:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575446.338625:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575446.338630:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.338661:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810419072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575446.338671:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575446.338673:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.338679:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810419072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575446.338697:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575446.338703:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15923) 00000800:00000200:0.0:1788575447.298036:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00080000:0.0:1788575447.362469:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575447.362474:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575447.362482:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575447.362485:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575447.362488:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575447.362491:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15918 00000400:00000200:0.0:1788575447.362499:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575447.362504:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575447.362507:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575447.362509:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362511:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362513:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362515:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575447.362517:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575447.362518:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575447.362519:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575447.362521:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575447.362522:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575447.362524:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575447.362526:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362527:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362529:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362530:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575447.362532:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575447.362534:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362535:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575447.362537:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575447.362539:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575447.362540:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362542:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575447.362543:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575447.362547:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575447.362548:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575447.362553:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575447.362555:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362557:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575447.362558:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575447.362560:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575447.362563:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575447.362566:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.362571:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810419328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575447.362576:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575447.362579:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.362582:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810419328/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575447.362599:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575447.362602:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575447.362604:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575447.362605:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575447.362606:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575447.362608:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575447.362610:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15918 00000400:00000200:0.0:1788575447.362612:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575447.362613:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575447.362615:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575447.362617:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362618:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362619:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362621:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575447.362622:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575447.362623:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575447.362624:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575447.362626:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575447.362630:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575447.362632:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575447.362634:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362662:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362665:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362667:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575447.362670:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575447.362673:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362676:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575447.362683:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575447.362692:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575447.362696:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362699:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575447.362705:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575447.362710:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362712:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575447.362716:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575447.362719:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575447.362721:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575447.362723:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362724:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575447.362725:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575447.362727:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575447.362731:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575447.362747:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.362758:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810419456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575447.362778:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575447.362782:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.362789:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810419456/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575447.362831:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575447.362835:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575447.362839:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575447.362842:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575447.362844:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575447.362850:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575447.362854:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15918 00000400:00000200:0.0:1788575447.362862:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575447.362868:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575447.362871:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575447.362873:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362879:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362884:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362888:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575447.362891:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575447.362895:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575447.362898:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575447.362900:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575447.362901:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575447.362903:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575447.362911:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362913:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.362914:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362920:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575447.362924:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575447.362929:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575447.362932:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575447.362938:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575447.362942:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575447.362945:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362950:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575447.362954:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575447.362961:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362968:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575447.362975:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575447.362981:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575447.362987:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575447.362989:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.362992:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575447.362993:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575447.362996:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575447.363004:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575447.363009:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.363015:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810419584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575447.363032:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575447.363034:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.363043:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810419584/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575447.363051:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575447.363053:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575447.363060:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575447.363068:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575447.363072:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575447.363078:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575447.363084:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15918 00000400:00000200:0.0:1788575447.363094:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575447.363100:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575447.363103:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575447.363113:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363115:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.363116:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363122:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575447.363126:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575447.363129:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575447.363134:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575447.363137:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575447.363143:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575447.363147:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575447.363159:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363166:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.363168:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363169:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575447.363171:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575447.363175:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363182:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575447.363185:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575447.363189:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575447.363193:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363201:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575447.363206:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575447.363208:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363211:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575447.363217:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575447.363222:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575447.363225:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575447.363226:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363227:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575447.363229:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575447.363233:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575447.363245:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575447.363257:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.363265:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810419712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575447.363279:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575447.363283:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.363290:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810419712/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575447.363305:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575447.363311:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575447.363314:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575447.363316:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575447.363318:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575447.363324:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575447.363328:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15918 00000400:00000200:0.0:1788575447.363335:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575447.363343:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575447.363345:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575447.363348:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363356:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.363359:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363368:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575447.363371:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575447.363374:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575447.363376:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575447.363377:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575447.363381:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575447.363383:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575447.363392:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363398:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575447.363405:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363408:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575447.363410:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575447.363419:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575447.363424:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575447.363426:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575447.363430:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575447.363440:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363450:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575447.363454:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575447.363461:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363464:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575447.363475:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575447.363480:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575447.363483:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575447.363485:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.363490:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575447.363491:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575447.363495:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575447.363504:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575447.363508:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.363518:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810419840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575447.363533:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575447.363536:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.363541:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810419840/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575447.363554:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575447.363557:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15924) 00000020:00000001:0.0:1788575449.281843:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575449.281846:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575449.281848:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575451.201764:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310590175, at 4310590208 00000100:00000010:0.0:1788575451.201780:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788575454.401813:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575454.401816:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575454.401818:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575456.321735:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310595294, at 4310595328 00000100:00000010:0.0:1788575456.321752:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 00000020:00000001:0.0:1788575459.521866:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575459.521870:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575459.521872:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788575460.674048:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788575460.674109:0:2392:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788575460.674121:0:2392:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575460.674124:0:2392:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788575460.674128:0:2392:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575460.674132:0:2392:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788575460.674137:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575460.674141:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f500. 00000400:00000200:0.0:1788575460.674147:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575460.674150:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575460.674158:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf17f8 00000400:00000010:0.0:1788575460.674161:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf17f8. 00000100:00000001:0.0:1788575460.674166:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575460.674175:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70380 x1875440810420608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674199:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575460.674204:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70380 x1875440810420608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674212:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575460.674214:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575460.674225:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575460.674246:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fe00. 00000400:00000200:0.0:1788575460.674249:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575460.674253:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575460.674256:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1e58 00000400:00000010:0.0:1788575460.674258:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1e58. 00000100:00000001:0.0:1788575460.674261:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575460.674265:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec71c00 x1875440810420736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674274:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575460.674277:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810420736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674284:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575460.674285:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575460.674290:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575460.674292:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fc00. 00000400:00000200:0.0:1788575460.674295:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575460.674298:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575460.674300:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf15d8 00000400:00000010:0.0:1788575460.674301:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf15d8. 00000100:00000001:0.0:1788575460.674304:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575460.674308:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70e00 x1875440810420864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674316:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575460.674319:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70e00 x1875440810420864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674326:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575460.674327:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575460.674331:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575460.674333:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f700. 00000400:00000200:0.0:1788575460.674335:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575460.674338:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575460.674345:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1110 00000400:00000010:0.0:1788575460.674347:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1110. 00000100:00000001:0.0:1788575460.674349:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575460.674353:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec70000 x1875440810420992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674361:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575460.674364:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec70000 x1875440810420992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674370:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575460.674372:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575460.674377:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575460.674379:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f600. 00000400:00000200:0.0:1788575460.674382:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575460.674399:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575460.674402:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1cc0 00000400:00000010:0.0:1788575460.674404:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1cc0. 00000100:00000001:0.0:1788575460.674407:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575460.674412:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ec72a00 x1875440810421120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674419:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575460.674423:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810421120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575460.674429:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575460.674430:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788575461.441792:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310600414, at 4310600448 00000100:00000010:0.0:1788575461.441809:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000100:00080000:0.0:1788575462.721830:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575462.721838:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575462.721840:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575462.721842:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575462.721845:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575462.721862:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15929 00000400:00000200:0.0:1788575462.721869:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575462.721874:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575462.721876:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575462.721878:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.721880:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.721881:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.721883:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575462.721886:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575462.721887:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575462.721889:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575462.721891:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575462.721893:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575462.721895:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575462.721897:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.721900:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.721905:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.721907:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575462.721910:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575462.721913:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575462.721915:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.721918:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575462.721921:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.721924:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.721926:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575462.721928:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575462.721935:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575462.721938:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575462.721941:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575462.721943:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.721946:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575462.721948:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575462.721958:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575462.721965:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575462.721972:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.721981:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810421248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575462.721991:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575462.721994:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.721998:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810421248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.722023:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575462.722026:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575462.722029:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575462.722030:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575462.722032:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575462.722034:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575462.722037:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15929 00000400:00000200:0.0:1788575462.722041:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575462.722043:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575462.722044:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575462.722046:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722048:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722049:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722050:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575462.722052:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575462.722053:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575462.722054:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575462.722056:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575462.722057:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575462.722058:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575462.722060:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722067:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722069:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722072:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575462.722075:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575462.722080:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722082:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.722085:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575462.722091:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.722096:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722099:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575462.722103:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575462.722109:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722114:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575462.722120:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575462.722125:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575462.722130:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575462.722133:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722137:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575462.722142:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575462.722146:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575462.722153:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575462.722159:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.722168:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810421376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575462.722181:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575462.722185:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.722192:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810421376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.722205:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575462.722209:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575462.722217:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575462.722226:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575462.722229:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575462.722235:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575462.722242:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15929 00000400:00000200:0.0:1788575462.722251:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575462.722258:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575462.722263:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575462.722267:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722270:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722275:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722278:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575462.722282:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575462.722285:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575462.722288:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575462.722292:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575462.722297:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575462.722302:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575462.722307:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722310:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722315:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722318:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575462.722323:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575462.722330:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722335:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.722338:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575462.722341:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.722346:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722349:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575462.722353:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575462.722359:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722367:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575462.722373:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575462.722377:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575462.722381:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575462.722382:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722383:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575462.722385:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575462.722387:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575462.722390:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575462.722394:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.722404:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810421504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575462.722419:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575462.722424:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.722431:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810421504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.722446:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575462.722455:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575462.722460:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575462.722463:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575462.722464:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575462.722468:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575462.722473:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15929 00000400:00000200:0.0:1788575462.722482:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575462.722487:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575462.722490:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575462.722501:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722505:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722509:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722514:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575462.722516:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575462.722522:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575462.722524:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575462.722526:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575462.722528:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575462.722530:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575462.722533:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722536:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722537:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722539:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575462.722541:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575462.722543:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722546:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.722547:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575462.722550:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.722552:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722553:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575462.722555:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575462.722558:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722559:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575462.722562:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575462.722564:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575462.722567:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575462.722568:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722570:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575462.722571:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575462.722573:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575462.722577:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575462.722580:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.722589:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810421632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575462.722599:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575462.722603:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.722619:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810421632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.722631:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575462.722679:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575462.722688:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575462.722692:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575462.722695:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575462.722698:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575462.722704:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15929 00000400:00000200:0.0:1788575462.722711:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575462.722718:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575462.722722:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575462.722728:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722734:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722737:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722742:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575462.722746:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575462.722749:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575462.722752:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575462.722756:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575462.722760:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575462.722775:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575462.722779:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722782:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575462.722788:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722791:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575462.722796:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575462.722802:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575462.722807:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.722818:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575462.722823:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.722828:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722832:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575462.722837:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575462.722843:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722846:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575462.722853:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575462.722856:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575462.722859:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575462.722861:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.722865:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575462.722868:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575462.722871:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575462.722875:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575462.722879:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.722885:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810421760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575462.722898:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575462.722902:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.722910:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810421760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.722920:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.722927:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15939) 00000020:00000001:0.0:1788575464.641869:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575464.641873:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575464.641876:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575466.561724:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310605534, at 4310605568 00000100:00000010:0.0:1788575466.561738:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 10000000:00000001:0.0:1788575467.841788:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575467.841794:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9780. 00000020:00000010:0.0:1788575467.841800:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575467.841805:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ba00. 00000040:00000010:0.0:1788575467.841814:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575467.841819:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575467.841825:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575467.841829:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575467.841835:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575467.841841:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985d600. 10000000:00000040:0.0:1788575467.841848:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575467.841850:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575467.841857:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ba00. 00000040:00000010:0.0:1788575467.841866:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575467.841872:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575467.841881:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575467.841888:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575467.841894:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575467.841898:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985d600. 00000020:00000010:0.0:1788575467.841903:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b600. 10000000:00000010:0.0:1788575467.841907:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9780. 10000000:01000000:0.0:1788575467.841941:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575467.841947:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575467.841951:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575467.841954:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575467.841959:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575467.841961:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575467.841963:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575467.841967:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575467.841968:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575467.841970:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575467.841974:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575467.841976:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575467.841980:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575469.313718:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575469.313729:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575469.313735:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575469.313738:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575469.313740:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575469.313759:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575469.313761:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575469.313763:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575469.313765:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575469.313767:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575469.313769:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575469.313777:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575469.313781:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575469.313786:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575469.313789:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575469.313795:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575469.313801:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985d600. 00000100:00000001:0.0:1788575469.313809:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575469.313813:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575469.313816:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575469.313823:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692400. 00010000:00000001:0.0:1788575469.313827:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575469.313831:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788575469.313835:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575469.313838:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168d67 to hash 00000020:00000001:0.0:1788575469.313840:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575469.313841:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575469.313843:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575469.313846:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575469.313849:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575469.313852:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575469.313861:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575469.313872:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575469.313877:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575469.313882:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575469.313888:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575469.313889:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575469.313892:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575469.313893:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575469.313898:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575469.313902:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575469.313904:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575469.313905:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575469.313915:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575469.313923:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575469.313926:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575469.313930:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575469.313933:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575469.313937:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575469.313939:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575469.313940:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575469.313942:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575469.313944:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575469.313946:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575469.313949:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575469.313950:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575469.313966:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575469.313970:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575469.313976:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810422528:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575469.313980:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575469.313986:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575469.313988:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575469.313990:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575469.313993:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575469.313999:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575469.314001:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575469.314005:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575469.314006:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575469.314008:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575469.314011:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575469.314013:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575469.314015:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788575469.314017:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575469.314019:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788575469.314022:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314026:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575469.314029:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314034:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575469.314037:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314041:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575469.314042:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575469.314044:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314047:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575469.314052:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575469.314054:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575469.314057:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575469.314062:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575469.314063:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575469.314065:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314069:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575469.314070:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575469.314071:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575469.314072:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788575469.314073:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575469.314074:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575469.314076:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575469.314078:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575469.314083:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575469.314085:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575469.314089:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575469.314091:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575469.314093:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575469.314096:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575469.314097:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575469.314101:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575469.314103:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575469.314105:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575469.314107:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575469.314109:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168d67 from hash 00010000:00000001:0.0:1788575469.314116:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575469.314117:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575469.314119:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575469.314120:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575469.314122:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575469.314124:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575469.314126:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575469.314127:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575469.314129:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575469.314131:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575469.314135:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575469.314142:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575469.314143:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575469.314145:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575469.314147:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575469.314149:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575469.314150:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575469.314151:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575469.314153:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168d67 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575469.314159:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692400 count: 0 00010000:00000001:0.0:1788575469.314167:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575469.314168:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575469.314170:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575469.314174:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810422528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575469.314179:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575469.314182:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575469.314184:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575469.314185:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575469.314188:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985d600. 02000000:00000001:0.0:1788575469.314191:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575469.314193:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575469.314201:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575469.314203:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575469.314204:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575469.314206:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575469.314208:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575469.314210:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575469.761814:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575469.761820:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575469.761823:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575473.153750:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310612127, at 4310612160 00000100:00000010:0.0:1788575473.153769:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 00000020:00000001:0.0:1788575474.881856:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575474.881860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575474.881863:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575478.337739:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310617310, at 4310617344 00000100:00000010:0.0:1788575478.337752:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000020:00000001:0.0:1788575480.001885:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575480.001889:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575480.001893:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575483.457755:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310622431, at 4310622464 00000100:00000010:0.0:1788575483.457773:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788575485.121866:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575485.121871:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575485.121875:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575487.233937:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575488.577735:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310627550, at 4310627584 00000100:00000010:0.0:1788575488.577747:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 10000000:00000001:0.0:1788575489.345780:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575489.345793:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9840. 00000020:00000010:0.0:1788575489.345801:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985d600. 00000020:00000010:0.0:1788575489.345807:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985dc00. 00000040:00000010:0.0:1788575489.345811:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575489.345816:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575489.345834:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575489.345838:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575489.345843:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575489.345846:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788575489.345851:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575489.345853:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575489.345858:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985dc00. 00000040:00000010:0.0:1788575489.345866:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575489.345870:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575489.345873:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575489.345876:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575489.345879:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575489.345882:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575489.345884:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985d600. 10000000:00000010:0.0:1788575489.345887:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9840. 10000000:01000000:0.0:1788575489.345890:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575489.345893:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575489.345895:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575489.345897:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575489.345899:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575489.345902:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575489.345903:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575489.345906:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575489.345908:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575489.345910:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575489.345913:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575489.345914:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575489.345917:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1788575490.241870:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575490.241874:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575490.241877:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575490.817709:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575490.817718:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575490.817722:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575490.817725:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575490.817741:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575490.817743:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575490.817745:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575490.817747:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575490.817749:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575490.817752:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575490.817753:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575490.817761:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575490.817766:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575490.817771:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575490.817774:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575490.817780:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575490.817787:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788575490.817794:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575490.817799:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575490.817802:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575490.817809:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692f00. 00010000:00000001:0.0:1788575490.817813:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575490.817818:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788575490.817822:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575490.817824:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168d6e to hash 00000020:00000001:0.0:1788575490.817827:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575490.817828:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575490.817832:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575490.817835:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575490.817837:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575490.817840:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575490.817848:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575490.817856:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575490.817867:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575490.817869:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575490.817873:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575490.817874:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575490.817876:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575490.817877:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575490.817880:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575490.817883:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575490.817885:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575490.817886:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575490.817893:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575490.817899:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575490.817902:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575490.817906:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575490.817908:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575490.817912:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575490.817913:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575490.817914:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575490.817916:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575490.817918:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575490.817920:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575490.817923:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575490.817925:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575490.817927:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575490.817931:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575490.817938:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810425856:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575490.817949:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575490.817953:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575490.817955:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575490.817957:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575490.817961:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575490.817968:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575490.817971:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575490.817976:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575490.817977:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575490.817980:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575490.817983:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575490.817985:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575490.817987:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788575490.817989:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575490.817991:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788575490.817994:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.817999:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575490.818002:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.818006:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575490.818007:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.818011:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575490.818012:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575490.818013:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.818024:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575490.818025:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575490.818027:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575490.818029:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575490.818032:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575490.818034:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575490.818035:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.818039:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575490.818039:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575490.818040:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575490.818041:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788575490.818042:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575490.818043:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575490.818045:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575490.818047:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575490.818051:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575490.818052:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575490.818055:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575490.818056:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575490.818057:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575490.818059:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575490.818060:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575490.818062:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575490.818065:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575490.818066:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575490.818067:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575490.818069:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168d6e from hash 00010000:00000001:0.0:1788575490.818071:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575490.818072:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575490.818073:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575490.818073:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575490.818074:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575490.818075:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575490.818076:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575490.818077:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575490.818078:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575490.818079:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575490.818080:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575490.818083:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575490.818084:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575490.818085:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575490.818086:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575490.818087:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575490.818087:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575490.818088:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575490.818089:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168d6e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575490.818093:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692f00 count: 0 00010000:00000001:0.0:1788575490.818100:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575490.818101:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575490.818102:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575490.818105:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810425856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575490.818109:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575490.818111:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575490.818112:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575490.818116:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575490.818118:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788575490.818120:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575490.818122:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575490.818124:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575490.818127:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575490.818127:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575490.818128:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575490.818130:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575490.818132:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575494.657745:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310633631, at 4310633664 00000100:00000010:0.0:1788575494.657765:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000020:00000001:0.0:1788575495.361903:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575495.361908:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575495.361911:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575499.841775:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310638814, at 4310638848 00000100:00000010:0.0:1788575499.841794:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000020:00000001:0.0:1788575500.482001:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575500.482006:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575500.482010:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575504.961769:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310643934, at 4310643968 00000100:00000010:0.0:1788575504.961796:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788575505.601924:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575505.601930:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575505.601933:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575510.081727:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310649055, at 4310649088 00000100:00000010:0.0:1788575510.081741:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb3c0. 00000020:00000001:0.0:1788575510.721865:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575510.721869:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575510.721871:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575510.849742:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575510.849754:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a99c0. 00000020:00000010:0.0:1788575510.849760:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575510.849764:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575510.849776:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575510.849780:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575510.849783:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575510.849786:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575510.849790:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575510.849794:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985d600. 10000000:00000040:0.0:1788575510.849797:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575510.849799:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575510.849803:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575510.849807:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575510.849810:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575510.849812:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575510.849814:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575510.849817:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575510.849818:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985d600. 00000020:00000010:0.0:1788575510.849821:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b000. 10000000:00000010:0.0:1788575510.849822:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a99c0. 10000000:01000000:0.0:1788575510.849824:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575510.849826:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575510.849828:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575510.849829:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575510.849830:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575510.849832:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575510.849833:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575510.849835:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575510.849836:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575510.849838:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575510.849839:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575510.849841:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575510.849842:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575512.769731:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575512.769737:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575512.769744:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575512.769746:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575512.769749:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575512.769764:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575512.769766:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575512.769768:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575512.769769:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575512.769771:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575512.769773:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575512.769780:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575512.769785:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575512.769790:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575512.769793:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575512.769799:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575512.769807:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985d600. 00000100:00000001:0.0:1788575512.769814:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575512.769820:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575512.769823:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575512.769828:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692b00. 00010000:00000001:0.0:1788575512.769831:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575512.769833:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788575512.769836:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575512.769838:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168d75 to hash 00000020:00000001:0.0:1788575512.769839:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575512.769840:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575512.769842:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575512.769845:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575512.769847:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575512.769850:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575512.769856:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575512.769863:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575512.769867:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575512.769872:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575512.769877:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575512.769878:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575512.769880:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575512.769882:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575512.769886:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575512.769890:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575512.769892:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575512.769894:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575512.769903:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575512.769912:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575512.769916:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575512.769930:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575512.769933:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575512.769937:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575512.769939:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575512.769940:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575512.769942:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575512.769943:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575512.769945:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575512.769947:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575512.769949:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575512.769950:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575512.769952:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575512.769957:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810429184:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575512.769960:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575512.769966:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575512.769968:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575512.769969:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575512.769971:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575512.769976:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575512.769977:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575512.769981:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575512.769982:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575512.769984:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575512.769986:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575512.769988:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575512.769989:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788575512.769991:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575512.769992:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788575512.769995:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.769999:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575512.770002:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.770005:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575512.770007:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.770010:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575512.770011:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575512.770013:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.770016:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575512.770019:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575512.770021:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575512.770023:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575512.770027:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575512.770028:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575512.770034:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.770038:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575512.770039:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575512.770040:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575512.770041:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788575512.770042:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575512.770043:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575512.770045:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575512.770047:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575512.770050:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575512.770051:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575512.770054:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575512.770056:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575512.770056:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575512.770058:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575512.770059:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575512.770061:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575512.770062:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575512.770063:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575512.770064:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575512.770065:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168d75 from hash 00010000:00000001:0.0:1788575512.770069:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575512.770071:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575512.770071:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575512.770072:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575512.770073:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575512.770074:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575512.770075:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575512.770076:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575512.770077:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575512.770078:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575512.770080:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575512.770082:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575512.770083:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575512.770084:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575512.770085:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575512.770086:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575512.770086:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575512.770087:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575512.770088:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168d75 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575512.770092:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692b00 count: 0 00010000:00000001:0.0:1788575512.770098:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575512.770098:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575512.770100:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575512.770102:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810429184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575512.770106:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575512.770108:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575512.770109:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575512.770110:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575512.770112:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985d600. 02000000:00000001:0.0:1788575512.770119:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575512.770121:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575512.770125:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575512.770127:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575512.770128:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575512.770129:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575512.770131:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575512.770133:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575515.841830:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575515.841833:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575515.841836:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575516.609717:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310655583, at 4310655616 00000100:00000010:0.0:1788575516.609734:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788575520.961872:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575520.961877:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575520.961880:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575521.857744:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310660831, at 4310660864 00000100:00000010:0.0:1788575521.857758:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000800:00000200:0.0:1788575525.121934:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575526.081827:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575526.081831:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575526.081834:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575526.977737:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310665950, at 4310665984 00000100:00000010:0.0:1788575526.977760:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000100:00080000:0.0:1788575528.257772:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575528.257782:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575528.257785:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575528.257789:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575528.257792:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575528.257799:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15994 00000400:00000200:0.0:1788575528.257808:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575528.257813:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575528.257816:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575528.257819:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.257839:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.257842:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.257845:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575528.257848:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575528.257851:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575528.257853:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575528.257856:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575528.257859:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575528.257861:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575528.257864:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.257866:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.257867:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.257869:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575528.257872:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575528.257876:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575528.257879:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575528.257882:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575528.257886:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575528.257889:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.257891:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575528.257893:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575528.257899:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575528.257902:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575528.257906:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575528.257908:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.257910:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575528.257913:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575528.257917:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575528.257923:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575528.257931:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.257939:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810431232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575528.257960:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575528.257965:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.257970:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810431232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575528.258001:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575528.258005:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575528.258008:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575528.258009:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575528.258011:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575528.258014:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575528.258016:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15994 00000400:00000200:0.0:1788575528.258021:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575528.258024:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575528.258026:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575528.258029:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258031:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258033:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258035:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575528.258037:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575528.258039:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575528.258040:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575528.258043:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575528.258045:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575528.258047:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575528.258050:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258052:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258054:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258056:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575528.258064:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575528.258067:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258070:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575528.258071:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575528.258078:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575528.258080:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258082:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575528.258086:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575528.258089:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258091:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575528.258094:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575528.258097:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575528.258099:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575528.258101:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258102:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575528.258104:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575528.258106:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575528.258110:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575528.258113:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.258119:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810431360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575528.258126:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575528.258128:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.258132:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810431360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575528.258140:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575528.258142:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575528.258145:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575528.258147:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575528.258148:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575528.258151:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575528.258158:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15994 00000400:00000200:0.0:1788575528.258162:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575528.258165:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575528.258167:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575528.258170:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258172:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258174:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258176:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575528.258178:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575528.258179:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575528.258181:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575528.258183:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575528.258185:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575528.258187:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575528.258195:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258197:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258199:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258201:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575528.258206:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575528.258209:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258212:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575528.258214:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575528.258217:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575528.258219:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258221:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575528.258223:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575528.258225:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258227:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575528.258231:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575528.258233:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575528.258236:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575528.258242:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258244:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575528.258245:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575528.258247:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575528.258251:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575528.258255:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.258259:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810431488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575528.258267:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575528.258269:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.258273:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810431488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575528.258281:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575528.258284:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575528.258287:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575528.258289:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575528.258290:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575528.258293:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575528.258295:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15994 00000400:00000200:0.0:1788575528.258299:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575528.258302:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575528.258304:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575528.258307:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258309:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258311:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258313:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575528.258315:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575528.258316:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575528.258318:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575528.258319:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575528.258321:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575528.258340:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575528.258344:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258346:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258348:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258350:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575528.258352:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575528.258355:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258357:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575528.258359:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575528.258362:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575528.258364:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258366:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575528.258368:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575528.258370:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258372:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575528.258376:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575528.258378:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575528.258380:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575528.258381:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258383:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575528.258384:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575528.258386:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575528.258390:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575528.258393:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.258398:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810431616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575528.258406:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575528.258408:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.258412:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810431616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575528.258420:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575528.258427:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575528.258430:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575528.258432:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575528.258433:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575528.258436:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575528.258439:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15994 00000400:00000200:0.0:1788575528.258443:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575528.258446:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575528.258448:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575528.258451:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258453:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258455:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258457:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575528.258459:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575528.258460:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575528.258461:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575528.258463:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575528.258465:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575528.258467:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575528.258480:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258482:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575528.258484:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258485:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575528.258488:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575528.258490:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575528.258492:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575528.258494:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575528.258497:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575528.258499:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258501:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575528.258508:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575528.258510:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258512:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575528.258516:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575528.258519:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575528.258522:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575528.258523:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575528.258525:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575528.258526:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575528.258528:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575528.258531:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575528.258535:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.258540:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810431744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575528.258547:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575528.258549:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.258553:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810431744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575528.258560:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575528.258565:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16004) 00000020:00000001:0.0:1788575531.201911:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575531.201916:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575531.201920:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575532.097808:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310671071, at 4310671104 00000100:00000010:0.0:1788575532.097826:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb540. 10000000:00000001:0.0:1788575532.865737:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575532.865755:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e80. 00000020:00000010:0.0:1788575532.865763:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985dc00. 00000020:00000010:0.0:1788575532.865769:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985d600. 00000040:00000010:0.0:1788575532.865774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575532.865779:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575532.865784:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575532.865788:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575532.865808:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575532.865812:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788575532.865818:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575532.865821:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575532.865827:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985d600. 00000040:00000010:0.0:1788575532.865838:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575532.865843:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575532.865847:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575532.865851:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575532.865855:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575532.865858:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575532.865861:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985dc00. 10000000:00000010:0.0:1788575532.865864:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e80. 10000000:01000000:0.0:1788575532.865867:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575532.865871:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575532.865874:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575532.865876:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575532.865878:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575532.865881:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575532.865882:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575532.865885:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575532.865887:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575532.865889:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575532.865891:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575532.865893:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575532.865895:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788575533.377802:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575533.377814:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575533.377817:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575533.377822:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575533.377827:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575533.377836:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 15999 00000400:00000200:0.0:1788575533.377848:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575533.377856:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575533.377891:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575533.377896:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.377901:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.377904:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.377906:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575533.377910:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575533.377913:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575533.377916:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575533.377919:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575533.377922:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575533.377924:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575533.377927:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.377929:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.377930:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.377932:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575533.377935:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575533.377939:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575533.377942:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575533.377946:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575533.377950:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575533.377953:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.377956:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575533.377959:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575533.377967:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575533.377972:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575533.377976:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575533.377978:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.377982:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575533.377985:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575533.377991:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575533.378000:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575533.378008:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.378026:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810431872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575533.378038:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575533.378043:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.378048:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810431872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575533.378074:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575533.378077:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575533.378081:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575533.378083:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575533.378085:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575533.378088:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575533.378091:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15999 00000400:00000200:0.0:1788575533.378095:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575533.378099:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575533.378102:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575533.378106:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378109:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378111:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378114:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575533.378117:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575533.378121:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575533.378124:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575533.378127:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575533.378129:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575533.378132:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575533.378145:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378151:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378156:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378167:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575533.378172:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575533.378178:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378183:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575533.378188:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575533.378196:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575533.378201:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378206:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575533.378213:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575533.378218:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378225:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575533.378232:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575533.378237:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575533.378243:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575533.378245:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378249:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575533.378250:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575533.378254:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575533.378263:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575533.378268:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.378277:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810432000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575533.378295:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575533.378299:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.378314:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810432000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575533.378331:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575533.378335:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575533.378339:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575533.378342:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575533.378345:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575533.378355:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575533.378362:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15999 00000400:00000200:0.0:1788575533.378367:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575533.378370:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575533.378372:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575533.378374:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378378:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378382:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378386:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575533.378390:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575533.378392:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575533.378394:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575533.378396:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575533.378401:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575533.378415:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575533.378421:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378423:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378428:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378433:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575533.378439:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575533.378446:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378450:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575533.378455:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575533.378459:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575533.378463:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378465:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575533.378469:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575533.378475:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378479:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575533.378486:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575533.378495:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575533.378500:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575533.378503:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378507:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575533.378510:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575533.378514:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575533.378520:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575533.378526:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.378535:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810432128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575533.378547:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575533.378552:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.378560:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810432128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575533.378577:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575533.378590:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575533.378598:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575533.378601:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575533.378604:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575533.378610:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575533.378617:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15999 00000400:00000200:0.0:1788575533.378625:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575533.378631:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575533.378671:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575533.378676:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378679:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378682:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378685:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575533.378687:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575533.378689:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575533.378698:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575533.378703:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575533.378707:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575533.378712:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575533.378720:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378724:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378726:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378730:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575533.378735:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575533.378742:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378748:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575533.378752:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575533.378755:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575533.378759:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378763:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575533.378767:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575533.378773:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378777:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575533.378781:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575533.378786:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575533.378789:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575533.378792:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378794:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575533.378796:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575533.378799:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575533.378805:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575533.378810:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.378813:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810432256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575533.378821:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575533.378823:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.378830:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810432256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575533.378841:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575533.378845:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575533.378852:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575533.378854:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575533.378857:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575533.378859:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575533.378864:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 15999 00000400:00000200:0.0:1788575533.378870:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575533.378875:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575533.378878:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575533.378881:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378884:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378887:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378891:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575533.378894:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575533.378895:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575533.378897:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575533.378900:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575533.378904:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575533.378906:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575533.378910:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378912:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575533.378916:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378917:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575533.378921:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575533.378925:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575533.378930:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575533.378936:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575533.378940:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575533.378943:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378946:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575533.378950:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575533.378955:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378957:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575533.378961:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575533.378963:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575533.378966:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575533.378969:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575533.378970:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575533.378972:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575533.378973:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575533.378976:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575533.378978:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.378984:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810432384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575533.378994:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575533.378997:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.379001:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810432384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575533.379015:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575533.379020:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16010) 10000000:00000001:0.0:1788575534.657810:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575534.657821:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575534.657826:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575534.657829:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575534.657831:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575534.657833:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575534.657836:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575534.657838:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575534.657841:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575534.657845:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575534.657869:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575534.657880:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575534.657886:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575534.657892:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575534.657895:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575534.657902:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575534.657912:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575534.657920:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575534.657926:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575534.657929:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575534.657938:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692100. 00010000:00000001:0.0:1788575534.657944:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575534.657948:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788575534.657954:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575534.657957:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x1441586093168d7c to hash 00000020:00000001:0.0:1788575534.657960:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575534.657961:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788575534.657965:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788575534.657969:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575534.657973:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575534.657978:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575534.657989:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575534.658002:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575534.658010:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575534.658012:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575534.658020:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575534.658023:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610432 : -109714523941184 : ffff9c3718c616c0) 00000100:00000001:0.0:1788575534.658026:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.658033:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788575534.658039:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575534.658045:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.658047:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575534.658050:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.658062:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.658078:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575534.658082:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.658090:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575534.658095:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.658101:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575534.658105:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575534.658107:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.658110:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c616c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788575534.658113:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575534.658116:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575534.658120:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.658123:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.658125:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.658130:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.658139:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810432512:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575534.658145:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.658148:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575534.658151:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575534.658153:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.658157:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.658176:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575534.658179:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c616c0. 00000100:00000001:0.0:1788575534.658186:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575534.658187:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575534.658191:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575534.658195:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575534.658197:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575534.658199:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=4 00000020:00000001:0.0:1788575534.658201:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788575534.658204:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00010000:0.0:1788575534.658208:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658215:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575534.658219:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658226:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575534.658229:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658236:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575534.658238:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575534.658241:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658255:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575534.658257:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575534.658260:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575534.658263:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575534.658281:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31e00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575534.658284:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575534.658287:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658293:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575534.658295:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575534.658296:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575534.658299:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=5 00000020:00000001:0.0:1788575534.658301:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788575534.658303:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788575534.658306:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575534.658309:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575534.658315:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575534.658318:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575534.658324:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575534.658327:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575534.658328:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575534.658331:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575534.658334:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575534.658337:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575534.658339:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575534.658341:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575534.658343:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575534.658345:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31e00 with handle 0x1441586093168d7c from hash 00010000:00000001:0.0:1788575534.658347:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575534.658349:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575534.658350:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575534.658356:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575534.658358:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575534.658360:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575534.658363:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575534.658364:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575534.658366:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575534.658367:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575534.658371:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575534.658376:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575534.658378:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575534.658379:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575534.658381:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575534.658382:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575534.658384:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575534.658385:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575534.658387:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168d7c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575534.658395:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000001:0.0:1788575534.658407:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575534.658409:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575534.658412:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.658416:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810432512/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.658423:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575534.658427:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.658428:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575534.658431:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.658434:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575534.658439:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.658441:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.658443:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788575534.658447:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.658448:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575534.658456:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575534.658458:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575534.658461:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575536.321865:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575536.321871:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575536.321874:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575538.497769:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310677471, at 4310677504 00000100:00000010:0.0:1788575538.497782:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb120. 00000020:00000001:0.0:1788575541.441822:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575541.441825:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575541.441828:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575543.873731:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310682846, at 4310682880 00000100:00000010:0.0:1788575543.873747:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000020:00000001:0.0:1788575546.561803:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575546.561805:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575546.561808:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575548.993741:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310687967, at 4310688000 00000100:00000010:0.0:1788575548.993755:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 00000020:00000001:0.0:1788575551.681851:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575551.681855:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575551.681858:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575554.113748:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310693086, at 4310693120 00000100:00000010:0.0:1788575554.113767:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbba80. 10000000:00000001:0.0:1788575554.881745:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575554.881757:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e00. 00000020:00000010:0.0:1788575554.881763:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575554.881767:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ba00. 00000040:00000010:0.0:1788575554.881769:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575554.881773:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575554.881776:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575554.881779:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575554.881783:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575554.881787:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985dc00. 10000000:00000040:0.0:1788575554.881800:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575554.881802:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575554.881806:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ba00. 00000040:00000010:0.0:1788575554.881812:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575554.881813:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575554.881816:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575554.881818:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c37116262a0. 00040000:00000010:0.0:1788575554.881820:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575554.881823:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985dc00. 00000020:00000010:0.0:1788575554.881826:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788575554.881829:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e00. 10000000:01000000:0.0:1788575554.881832:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575554.881835:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575554.881837:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575554.881839:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575554.881840:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575554.881843:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575554.881845:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575554.881848:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575554.881850:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575554.881853:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575554.881855:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575554.881857:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575554.881859:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575556.417737:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575556.417744:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575556.417747:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575556.417749:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575556.417750:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575556.417751:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575556.417753:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575556.417754:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575556.417756:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575556.417758:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575556.417759:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575556.417768:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575556.417782:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575556.417786:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575556.417788:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575556.417792:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575556.417797:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788575556.417802:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575556.417805:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575556.417807:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575556.417812:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692800. 00010000:00000001:0.0:1788575556.417815:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575556.417817:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31800. 00000020:00000001:0.0:1788575556.417820:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575556.417823:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31800 with handle 0x1441586093168d83 to hash 00000020:00000001:0.0:1788575556.417824:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575556.417825:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788575556.417827:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788575556.417829:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575556.417831:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575556.417834:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575556.417840:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575556.417848:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575556.417852:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575556.417853:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575556.417857:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575556.417858:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575556.417860:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575556.417861:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788575556.417864:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575556.417870:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575556.417871:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575556.417873:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575556.417880:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575556.417886:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575556.417889:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575556.417893:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575556.417895:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575556.417899:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575556.417900:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575556.417901:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575556.417903:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575556.417904:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575556.417906:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575556.417908:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575556.417910:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575556.417911:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575556.417915:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575556.417922:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810435840:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575556.417927:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575556.417929:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575556.417932:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575556.417933:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575556.417937:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575556.417944:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575556.417952:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575556.417957:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575556.417958:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575556.417961:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575556.417964:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575556.417966:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575556.417968:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=4 00000020:00000001:0.0:1788575556.417970:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788575556.417972:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00010000:0.0:1788575556.417975:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.417981:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575556.417985:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.417990:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575556.417993:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.417999:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575556.418000:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575556.418003:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.418036:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575556.418038:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575556.418041:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575556.418044:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575556.418050:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575556.418057:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575556.418059:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.418066:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575556.418067:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575556.418068:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575556.418070:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31800 refcount=5 00000020:00000001:0.0:1788575556.418072:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788575556.418074:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687104 : -109714172864512 : ffff9c372db31800) 00010000:00000001:0.0:1788575556.418077:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575556.418080:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575556.418086:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575556.418088:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575556.418094:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575556.418096:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575556.418097:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575556.418100:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575556.418102:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575556.418104:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575556.418105:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575556.418106:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575556.418108:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575556.418109:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31800 with handle 0x1441586093168d83 from hash 00010000:00000001:0.0:1788575556.418111:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575556.418111:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575556.418112:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575556.418113:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575556.418114:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575556.418115:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575556.418117:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575556.418122:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575556.418123:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575556.418124:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575556.418127:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575556.418130:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575556.418131:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575556.418132:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575556.418133:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575556.418134:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575556.418134:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575556.418135:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575556.418136:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31800/0x1441586093168d83 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575556.418140:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692800 count: 0 00010000:00000001:0.0:1788575556.418147:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575556.418148:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575556.418150:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575556.418153:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810435840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575556.418156:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575556.418159:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575556.418160:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575556.418161:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575556.418163:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788575556.418166:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575556.418167:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575556.418169:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788575556.418171:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575556.418172:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575556.418173:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575556.418174:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575556.418176:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575556.801790:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575556.801809:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575556.801813:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575560.257777:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310699231, at 4310699264 00000100:00000010:0.0:1788575560.257797:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbae0. 00000100:00080000:0.0:1788575561.537790:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575561.537803:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575561.537806:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575561.537811:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575561.537816:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575561.537825:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16028 00000400:00000200:0.0:1788575561.537837:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575561.537845:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575561.537849:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575561.537853:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.537856:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.537858:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.537861:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575561.537866:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575561.537868:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575561.537871:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575561.537875:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575561.537877:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575561.537880:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575561.537883:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.537885:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.537887:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.537889:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575561.537892:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575561.537896:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575561.537899:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575561.537923:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575561.537927:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575561.537929:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.537932:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575561.537935:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575561.537944:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575561.537948:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575561.537953:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575561.537954:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.537957:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575561.537960:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575561.537965:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575561.537975:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575561.537983:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.537995:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810436608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575561.538006:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575561.538011:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.538016:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810436608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575561.538043:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575561.538047:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575561.538051:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575561.538052:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575561.538054:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575561.538057:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575561.538059:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16028 00000400:00000200:0.0:1788575561.538064:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575561.538067:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575561.538070:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575561.538073:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538076:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538100:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538104:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575561.538107:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575561.538111:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575561.538115:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575561.538118:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575561.538122:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575561.538127:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575561.538136:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538139:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538144:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538149:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575561.538152:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575561.538157:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538163:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575561.538166:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575561.538173:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575561.538178:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538181:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575561.538185:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575561.538188:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538191:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575561.538195:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575561.538200:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575561.538202:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575561.538204:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538205:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575561.538208:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575561.538213:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575561.538217:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575561.538222:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.538230:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810436736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575561.538245:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575561.538249:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.538253:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810436736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575561.538272:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575561.538276:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575561.538281:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575561.538284:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575561.538288:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575561.538293:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575561.538298:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16028 00000400:00000200:0.0:1788575561.538307:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575561.538315:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575561.538319:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575561.538327:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538332:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538337:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538341:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575561.538345:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575561.538349:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575561.538353:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575561.538356:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575561.538362:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575561.538368:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575561.538375:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538381:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538388:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538393:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575561.538405:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575561.538413:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538419:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575561.538421:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575561.538425:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575561.538431:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538435:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575561.538439:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575561.538444:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538448:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575561.538455:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575561.538460:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575561.538466:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575561.538469:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538473:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575561.538475:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575561.538478:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575561.538485:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575561.538490:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.538496:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810436864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575561.538506:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575561.538510:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.538514:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810436864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575561.538522:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575561.538526:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575561.538528:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575561.538530:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575561.538531:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575561.538536:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575561.538543:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16028 00000400:00000200:0.0:1788575561.538548:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575561.538553:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575561.538557:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575561.538562:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538563:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538564:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538567:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575561.538570:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575561.538571:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575561.538573:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575561.538575:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575561.538579:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575561.538585:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575561.538592:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538597:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538603:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538610:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575561.538614:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575561.538617:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538620:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575561.538622:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575561.538625:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575561.538627:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538629:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575561.538634:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575561.538677:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538683:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575561.538689:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575561.538693:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575561.538698:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575561.538705:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538708:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575561.538711:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575561.538714:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575561.538720:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575561.538728:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.538739:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810436992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575561.538749:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575561.538752:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.538757:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810436992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575561.538771:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575561.538777:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575561.538784:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575561.538788:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575561.538791:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575561.538795:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575561.538799:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16028 00000400:00000200:0.0:1788575561.538810:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575561.538818:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575561.538822:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575561.538825:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538829:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538832:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538838:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575561.538842:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575561.538845:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575561.538850:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575561.538854:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575561.538859:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575561.538871:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575561.538877:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538882:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575561.538894:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538898:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575561.538901:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575561.538909:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575561.538916:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575561.538922:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575561.538928:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575561.538935:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538940:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575561.538945:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575561.538952:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538956:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575561.538964:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575561.538967:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575561.538972:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575561.538975:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575561.538980:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575561.538983:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575561.538988:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575561.538996:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575561.539003:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.539014:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810437120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575561.539035:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575561.539040:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.539049:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810437120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575561.539082:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575561.539102:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16038) 00000020:00000001:0.0:1788575561.921862:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575561.921867:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575561.921871:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575563.010044:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575565.377774:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310704350, at 4310704384 00000100:00000010:0.0:1788575565.377784:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000100:00080000:0.0:1788575566.657760:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575566.657767:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575566.657770:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575566.657774:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575566.657777:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575566.657784:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16033 00000400:00000200:0.0:1788575566.657793:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575566.657798:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575566.657800:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575566.657803:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.657805:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.657807:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.657809:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575566.657813:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575566.657815:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575566.657818:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575566.657820:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575566.657822:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575566.657824:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575566.657827:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.657829:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.657831:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.657834:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575566.657836:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575566.657847:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575566.657849:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575566.657852:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575566.657855:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575566.657858:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.657860:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575566.657862:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575566.657867:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575566.657871:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575566.657874:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575566.657876:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.657879:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575566.657882:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575566.657886:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575566.657892:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575566.657899:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.657908:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810437248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575566.657918:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575566.657923:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.657928:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810437248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575566.657947:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575566.657951:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575566.657955:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575566.657957:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575566.657959:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575566.657962:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575566.657966:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16033 00000400:00000200:0.0:1788575566.657983:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575566.657993:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575566.657997:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575566.658007:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658010:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658014:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658018:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575566.658022:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575566.658024:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575566.658028:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575566.658029:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575566.658031:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575566.658035:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575566.658040:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658043:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658046:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658047:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575566.658050:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575566.658056:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658060:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575566.658061:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575566.658067:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575566.658071:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658076:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575566.658080:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575566.658085:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658090:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575566.658096:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575566.658101:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575566.658107:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575566.658110:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658116:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575566.658119:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575566.658124:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575566.658131:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575566.658140:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.658147:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810437376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575566.658157:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575566.658162:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.658182:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810437376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575566.658194:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575566.658198:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575566.658205:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575566.658207:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575566.658209:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575566.658216:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575566.658222:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16033 00000400:00000200:0.0:1788575566.658231:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575566.658239:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575566.658244:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575566.658253:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658259:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658264:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658270:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575566.658274:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575566.658276:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575566.658279:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575566.658283:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575566.658286:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575566.658288:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575566.658294:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658297:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658306:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658310:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575566.658314:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575566.658319:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658322:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575566.658325:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575566.658329:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575566.658333:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658337:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575566.658340:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575566.658343:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658346:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575566.658348:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575566.658353:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575566.658356:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575566.658357:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658359:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575566.658360:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575566.658362:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575566.658367:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575566.658371:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.658379:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810437504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575566.658386:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575566.658390:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.658395:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810437504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575566.658407:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575566.658410:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575566.658413:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575566.658416:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575566.658422:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575566.658425:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575566.658430:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16033 00000400:00000200:0.0:1788575566.658436:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575566.658442:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575566.658446:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575566.658452:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658457:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658460:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658463:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575566.658466:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575566.658468:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575566.658470:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575566.658473:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575566.658478:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575566.658481:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575566.658487:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658489:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658495:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658496:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575566.658501:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575566.658506:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658510:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575566.658513:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575566.658517:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575566.658520:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658522:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575566.658526:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575566.658530:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658532:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575566.658541:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575566.658545:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575566.658550:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575566.658553:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658556:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575566.658558:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575566.658561:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575566.658564:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575566.658566:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.658572:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810437632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575566.658594:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575566.658600:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.658608:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810437632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575566.658629:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575566.658634:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575566.658654:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575566.658655:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575566.658657:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575566.658660:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575566.658665:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16033 00000400:00000200:0.0:1788575566.658674:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575566.658681:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575566.658686:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575566.658694:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658697:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658700:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658703:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575566.658707:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575566.658714:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575566.658717:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575566.658719:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575566.658722:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575566.658726:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575566.658732:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658736:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575566.658740:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658743:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575566.658746:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575566.658750:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575566.658753:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575566.658756:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575566.658761:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575566.658766:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658769:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575566.658772:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575566.658775:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658778:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575566.658782:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575566.658784:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575566.658789:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575566.658791:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575566.658793:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575566.658794:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575566.658797:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575566.658800:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575566.658805:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.658812:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810437760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575566.658819:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575566.658824:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.658833:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810437760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575566.658842:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575566.658849:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16043) 00000020:00000001:0.0:1788575567.041797:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575567.041801:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575567.041804:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575570.497780:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310709470, at 4310709504 00000100:00000010:0.0:1788575570.497791:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000020:00000001:0.0:1788575572.161903:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575572.161908:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575572.161911:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575575.617776:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310714590, at 4310714624 00000100:00000010:0.0:1788575575.617793:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000020:00000001:0.0:1788575577.281833:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575577.281838:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575577.281841:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575582.273732:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310721247, at 4310721280 00000100:00000010:0.0:1788575582.273743:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788575582.401769:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575582.401772:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575582.401776:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575587.393783:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310726366, at 4310726400 00000100:00000010:0.0:1788575587.393802:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000020:00000001:0.0:1788575587.521856:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575587.521859:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575587.521861:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575592.513756:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310731486, at 4310731520 00000100:00000010:0.0:1788575592.513770:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000020:00000001:0.0:1788575592.641820:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575592.641823:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575592.641825:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575597.633720:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310736607, at 4310736640 00000100:00000010:0.0:1788575597.633737:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788575597.761919:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575597.761923:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575597.761926:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575601.921945:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575602.883874:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575602.883878:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575602.883881:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575604.289797:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310743263, at 4310743296 00000100:00000010:0.0:1788575604.289820:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788575608.001891:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575608.001899:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575608.001904:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575609.409747:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310748382, at 4310748416 00000100:00000010:0.0:1788575609.409770:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788575613.121888:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575613.121895:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575613.121899:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575614.529748:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310753502, at 4310753536 00000100:00000010:0.0:1788575614.529764:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788575618.241801:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575618.241805:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575618.241807:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575619.649755:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310758623, at 4310758656 00000100:00000010:0.0:1788575619.649764:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788575623.361881:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575623.361889:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575623.361894:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575626.369750:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310765343, at 4310765376 00000100:00000010:0.0:1788575626.369769:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788575627.585849:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575627.585876:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575627.585879:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575627.585882:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575627.585886:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575627.585893:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16094 00000400:00000200:0.0:1788575627.585903:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575627.585908:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575627.585911:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575627.585914:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.585917:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.585919:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.585921:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575627.585925:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575627.585927:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575627.585935:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575627.585939:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575627.585941:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575627.585944:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575627.585947:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.585950:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.585951:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.585952:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575627.585954:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575627.585957:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575627.585958:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575627.585960:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575627.585963:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575627.585965:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.585966:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575627.585968:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575627.585974:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575627.585983:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575627.585986:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575627.585988:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.585990:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575627.585993:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575627.585997:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575627.586005:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575627.586012:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575627.586020:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810446592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575627.586027:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575627.586031:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586036:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810446592/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575627.586059:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575627.586062:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575627.586065:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575627.586066:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575627.586067:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575627.586069:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575627.586070:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16094 00000400:00000200:0.0:1788575627.586073:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575627.586075:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575627.586077:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575627.586079:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586085:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586088:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586091:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575627.586093:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575627.586094:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575627.586096:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575627.586103:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575627.586106:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575627.586110:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575627.586114:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586116:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586118:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586121:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575627.586125:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575627.586130:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586133:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575627.586135:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575627.586140:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575627.586146:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586151:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575627.586157:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575627.586162:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586166:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575627.586173:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575627.586179:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575627.586185:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575627.586188:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586192:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575627.586194:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575627.586197:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575627.586202:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575627.586209:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575627.586216:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810446720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575627.586230:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575627.586233:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586239:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810446720/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575627.586270:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575627.586275:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575627.586281:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575627.586284:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575627.586287:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575627.586292:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575627.586297:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16094 00000400:00000200:0.0:1788575627.586303:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575627.586309:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575627.586313:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575627.586319:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586322:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586326:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586328:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575627.586333:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575627.586335:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575627.586338:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575627.586340:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575627.586344:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575627.586349:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575627.586353:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586356:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586358:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586360:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575627.586364:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575627.586370:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586376:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575627.586385:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575627.586390:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575627.586396:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586400:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575627.586402:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575627.586404:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586406:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575627.586409:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575627.586412:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575627.586415:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575627.586418:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586423:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575627.586425:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575627.586429:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575627.586435:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575627.586441:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575627.586461:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810446848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575627.586476:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575627.586481:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586488:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810446848/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575627.586506:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575627.586511:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575627.586517:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575627.586520:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575627.586523:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575627.586528:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575627.586532:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16094 00000400:00000200:0.0:1788575627.586540:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575627.586546:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575627.586551:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575627.586560:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586562:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586564:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586568:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575627.586571:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575627.586574:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575627.586577:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575627.586581:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575627.586586:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575627.586591:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575627.586597:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586600:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586605:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586609:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575627.586627:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575627.586632:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586662:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575627.586667:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575627.586670:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575627.586672:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586676:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575627.586678:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575627.586681:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586686:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575627.586690:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575627.586695:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575627.586700:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575627.586703:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586706:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575627.586710:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575627.586715:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575627.586721:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575627.586731:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575627.586741:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810446976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575627.586754:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575627.586758:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586764:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810446976/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575627.586778:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575627.586783:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575627.586788:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575627.586790:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575627.586793:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575627.586798:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575627.586803:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16094 00000400:00000200:0.0:1788575627.586810:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575627.586814:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575627.586818:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575627.586823:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586825:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586830:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586834:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575627.586839:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575627.586841:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575627.586844:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575627.586848:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575627.586852:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575627.586855:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575627.586863:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586867:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575627.586877:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586879:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575627.586882:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575627.586886:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575627.586890:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575627.586893:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575627.586896:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575627.586899:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586902:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575627.586903:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575627.586906:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586907:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575627.586909:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575627.586911:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575627.586912:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575627.586913:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575627.586917:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575627.586920:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575627.586921:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575627.586925:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575627.586929:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575627.586938:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810447104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575627.586959:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575627.586963:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586969:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810447104/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575627.586977:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575627.586985:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16104) 00000020:00000001:0.0:1788575628.481824:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575628.481827:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575628.481830:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575631.425795:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310770398, at 4310770432 00000100:00000010:0.0:1788575631.425818:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788575633.601864:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575633.601869:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575633.601874:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575636.545807:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310775518, at 4310775552 00000100:00000010:0.0:1788575636.545833:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00080000:0.0:1788575637.825822:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575637.825831:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575637.825832:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575637.825835:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575637.825838:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575637.825844:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16104 00000400:00000200:0.0:1788575637.825852:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575637.825857:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575637.825859:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575637.825861:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.825863:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.825864:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.825866:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575637.825868:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575637.825870:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575637.825872:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575637.825874:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575637.825876:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575637.825877:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575637.825879:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.825880:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.825881:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.825882:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575637.825898:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575637.825900:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575637.825902:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575637.825904:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575637.825907:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575637.825908:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.825910:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575637.825911:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575637.825916:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575637.825918:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575637.825921:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575637.825922:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.825924:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575637.825927:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575637.825930:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575637.825944:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575637.825950:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.825959:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810447872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575637.825968:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575637.825971:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.825976:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810447872/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575637.826004:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575637.826006:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575637.826008:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575637.826009:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575637.826011:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575637.826012:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575637.826015:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16104 00000400:00000200:0.0:1788575637.826019:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575637.826036:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575637.826038:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575637.826039:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826041:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826042:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826043:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575637.826044:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575637.826045:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575637.826046:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575637.826047:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575637.826048:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575637.826050:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575637.826054:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826062:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826067:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826074:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575637.826079:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575637.826086:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826092:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575637.826098:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575637.826107:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575637.826113:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826118:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575637.826125:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575637.826131:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826132:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575637.826134:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575637.826148:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575637.826153:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575637.826157:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826164:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575637.826177:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575637.826182:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575637.826191:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575637.826206:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.826215:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810448000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575637.826231:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575637.826240:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.826249:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810448000/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575637.826311:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575637.826321:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575637.826326:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575637.826331:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575637.826334:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575637.826337:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575637.826345:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16104 00000400:00000200:0.0:1788575637.826387:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575637.826395:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575637.826400:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575637.826406:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826410:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826416:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826424:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575637.826431:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575637.826435:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575637.826442:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575637.826444:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575637.826450:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575637.826455:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575637.826471:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826475:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826481:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826482:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575637.826484:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575637.826488:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826494:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575637.826496:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575637.826500:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575637.826509:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826514:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575637.826519:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575637.826522:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826525:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575637.826531:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575637.826535:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575637.826537:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575637.826543:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826549:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575637.826557:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575637.826562:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575637.826573:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575637.826580:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.826586:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810448128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575637.826623:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575637.826628:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.826633:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810448128/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575637.826688:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575637.826698:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575637.826708:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575637.826711:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575637.826713:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575637.826715:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575637.826718:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16104 00000400:00000200:0.0:1788575637.826724:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575637.826727:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575637.826729:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575637.826732:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826741:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826748:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826754:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575637.826757:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575637.826765:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575637.826776:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575637.826778:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575637.826779:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575637.826780:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575637.826785:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826787:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.826792:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826798:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575637.826807:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575637.826818:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575637.826830:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575637.826832:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575637.826835:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575637.826840:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826842:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575637.826847:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575637.826860:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826867:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575637.826876:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575637.826883:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575637.826896:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575637.826901:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.826910:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575637.826915:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575637.826922:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575637.826931:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575637.826939:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.826950:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810448256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575637.826967:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575637.826972:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.826988:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810448256/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575637.827021:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575637.827031:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575637.827040:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575637.827043:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575637.827044:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575637.827046:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575637.827049:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16104 00000400:00000200:0.0:1788575637.827052:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575637.827054:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575637.827056:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575637.827058:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.827060:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.827062:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.827069:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575637.827074:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575637.827076:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575637.827080:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575637.827083:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575637.827090:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575637.827096:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575637.827106:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.827119:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575637.827124:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575637.827132:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575637.827135:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575637.827139:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575637.827144:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575637.827151:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575637.827158:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575637.827167:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.827173:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575637.827177:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575637.827184:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.827186:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575637.827191:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575637.827194:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575637.827202:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575637.827211:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.827215:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575637.827216:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575637.827219:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575637.827225:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575637.827230:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.827237:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810448384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575637.827263:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575637.827271:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.827283:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810448384/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575637.827301:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575637.827309:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16114) 00000020:00000001:0.0:1788575638.721860:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575638.721864:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575638.721868:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575639.809929:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575641.665765:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310780639, at 4310780672 00000100:00000010:0.0:1788575641.665780:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788575643.841863:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575643.841867:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575643.841870:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575647.937777:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310786911, at 4310786944 00000100:00000010:0.0:1788575647.937797:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000020:00000001:0.0:1788575648.961883:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575648.961887:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575648.961890:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575649.601812:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575649.601824:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575649.601826:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575649.601830:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575649.601834:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575649.601842:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16115 00000400:00000200:0.0:1788575649.601852:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575649.601858:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575649.601860:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575649.601863:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.601887:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.601889:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.601890:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575649.601895:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575649.601897:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575649.601900:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575649.601904:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575649.601907:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575649.601910:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575649.601913:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.601915:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.601917:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.601920:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575649.601923:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575649.601927:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575649.601930:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575649.601933:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575649.601938:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575649.601941:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.601943:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575649.601945:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575649.601952:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575649.601955:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575649.601959:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575649.601960:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.601963:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575649.601966:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575649.601970:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575649.601977:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575649.601984:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.601993:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810449920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575649.602010:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575649.602014:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602017:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810449920/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575649.602038:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575649.602041:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575649.602043:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575649.602044:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575649.602046:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575649.602048:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575649.602049:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16115 00000400:00000200:0.0:1788575649.602052:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575649.602055:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575649.602056:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575649.602059:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602061:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602062:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602064:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575649.602066:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575649.602067:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575649.602068:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575649.602070:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575649.602072:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575649.602073:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575649.602075:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602076:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602078:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602079:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575649.602086:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575649.602089:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602093:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575649.602095:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575649.602101:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575649.602106:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602109:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575649.602115:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575649.602118:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602122:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575649.602126:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575649.602128:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575649.602130:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575649.602133:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602135:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575649.602137:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575649.602141:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575649.602148:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575649.602152:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.602159:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810450048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575649.602168:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575649.602171:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602186:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810450048/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575649.602202:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575649.602206:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575649.602211:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575649.602213:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575649.602218:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575649.602224:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575649.602237:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16115 00000400:00000200:0.0:1788575649.602244:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575649.602250:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575649.602254:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575649.602258:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602263:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602268:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602269:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575649.602271:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575649.602272:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575649.602274:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575649.602277:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575649.602279:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575649.602284:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575649.602292:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602298:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602304:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602307:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575649.602311:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575649.602317:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602324:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575649.602326:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575649.602328:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575649.602329:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602331:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575649.602333:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575649.602339:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602342:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575649.602346:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575649.602350:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575649.602352:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575649.602358:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602362:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575649.602364:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575649.602366:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575649.602371:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575649.602377:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.602382:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810450176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575649.602389:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575649.602393:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602398:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810450176/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575649.602406:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575649.602409:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575649.602415:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575649.602417:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575649.602418:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575649.602422:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575649.602426:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16115 00000400:00000200:0.0:1788575649.602441:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575649.602447:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575649.602453:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575649.602459:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602466:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602471:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602476:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575649.602482:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575649.602485:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575649.602487:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575649.602489:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575649.602493:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575649.602501:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575649.602507:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602511:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602513:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602517:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575649.602522:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575649.602525:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602527:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575649.602530:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575649.602534:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575649.602538:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602542:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575649.602545:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575649.602550:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602551:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575649.602554:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575649.602559:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575649.602560:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575649.602561:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602562:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575649.602563:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575649.602565:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575649.602569:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575649.602573:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.602577:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810450304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575649.602586:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575649.602590:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602593:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810450304/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575649.602601:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575649.602609:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575649.602614:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575649.602616:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575649.602618:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575649.602625:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575649.602631:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16115 00000400:00000200:0.0:1788575649.602667:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575649.602674:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575649.602688:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575649.602697:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602702:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602708:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602714:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575649.602721:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575649.602724:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575649.602726:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575649.602730:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575649.602734:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575649.602738:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575649.602745:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602750:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575649.602753:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602757:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575649.602761:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575649.602769:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575649.602776:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575649.602781:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575649.602786:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575649.602793:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602798:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575649.602806:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575649.602809:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602810:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575649.602813:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575649.602814:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575649.602817:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575649.602819:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.602823:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575649.602824:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575649.602827:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575649.602830:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575649.602835:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.602843:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810450432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575649.602856:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575649.602859:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602866:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810450432/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575649.602875:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575649.602881:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16126) 00000100:00000040:0.0:1788575653.441895:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310792414, at 4310792448 00000100:00000010:0.0:1788575653.441902:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000020:00000001:0.0:1788575654.081852:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575654.081856:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575654.081859:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575658.561773:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310797535, at 4310797568 00000100:00000010:0.0:1788575658.561787:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000020:00000001:0.0:1788575659.201859:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575659.201862:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575659.201865:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575663.681764:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310802655, at 4310802688 00000100:00000010:0.0:1788575663.681791:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000020:00000001:0.0:1788575664.321922:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575664.321930:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575664.321934:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575664.961756:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575664.961771:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575664.961774:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575664.961780:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575664.961785:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575664.961794:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16131 00000400:00000200:0.0:1788575664.961806:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575664.961813:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575664.961817:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575664.961821:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.961824:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.961827:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.961829:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575664.961833:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575664.961835:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575664.961837:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575664.961840:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575664.961842:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575664.961844:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575664.961846:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.961847:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.961848:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.961849:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575664.961851:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575664.961854:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575664.961856:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575664.961858:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575664.961881:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575664.961883:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.961885:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575664.961888:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575664.961894:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575664.961897:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575664.961901:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575664.961902:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.961905:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575664.961908:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575664.961912:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575664.961922:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575664.961929:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.961937:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810451840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575664.961946:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575664.961956:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.961960:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810451840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575664.961979:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575664.961981:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575664.961984:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575664.961985:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575664.961986:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575664.961988:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575664.961990:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16131 00000400:00000200:0.0:1788575664.961993:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575664.961995:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575664.961997:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575664.961999:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962000:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962009:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962011:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575664.962013:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575664.962014:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575664.962015:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575664.962017:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575664.962018:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575664.962020:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575664.962022:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962024:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962026:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962029:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575664.962031:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575664.962033:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962036:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575664.962038:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575664.962045:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575664.962047:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962049:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575664.962055:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575664.962060:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962063:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575664.962068:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575664.962070:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575664.962073:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575664.962076:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962078:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575664.962079:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575664.962081:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575664.962084:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575664.962090:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.962101:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810451968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575664.962119:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575664.962124:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.962133:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810451968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575664.962147:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575664.962151:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575664.962158:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575664.962162:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575664.962166:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575664.962171:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575664.962176:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16131 00000400:00000200:0.0:1788575664.962186:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575664.962194:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575664.962198:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575664.962205:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962209:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962212:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962214:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575664.962216:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575664.962218:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575664.962221:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575664.962226:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575664.962229:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575664.962231:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575664.962237:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962241:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962243:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962248:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575664.962268:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575664.962276:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962282:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575664.962287:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575664.962291:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575664.962297:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962300:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575664.962303:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575664.962310:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962316:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575664.962322:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575664.962327:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575664.962332:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575664.962334:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962339:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575664.962342:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575664.962347:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575664.962355:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575664.962370:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.962381:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810452096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575664.962394:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575664.962397:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.962407:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810452096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575664.962429:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575664.962439:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575664.962445:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575664.962448:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575664.962452:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575664.962458:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575664.962469:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16131 00000400:00000200:0.0:1788575664.962484:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575664.962489:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575664.962491:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575664.962497:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962501:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962506:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962509:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575664.962520:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575664.962523:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575664.962525:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575664.962528:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575664.962533:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575664.962539:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575664.962547:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962551:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962557:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962562:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575664.962568:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575664.962573:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962578:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575664.962582:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575664.962588:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575664.962593:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962599:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575664.962604:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575664.962611:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962623:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575664.962630:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575664.962667:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575664.962672:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575664.962693:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962699:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575664.962703:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575664.962706:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575664.962714:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575664.962720:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.962730:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810452224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575664.962748:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575664.962754:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.962765:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810452224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575664.962787:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575664.962792:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575664.962799:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575664.962804:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575664.962808:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575664.962812:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575664.962818:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16131 00000400:00000200:0.0:1788575664.962829:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575664.962835:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575664.962839:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575664.962848:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962854:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962861:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962867:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575664.962873:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575664.962877:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575664.962881:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575664.962887:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575664.962894:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575664.962905:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575664.962914:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962921:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575664.962927:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962933:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575664.962939:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575664.962948:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575664.962955:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575664.962960:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575664.962966:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575664.962973:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962978:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575664.962983:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575664.962991:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.962996:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575664.963002:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575664.963008:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575664.963015:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575664.963017:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.963020:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575664.963022:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575664.963026:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575664.963031:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575664.963037:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.963045:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810452352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575664.963057:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575664.963061:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.963071:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810452352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575664.963102:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575664.963116:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16141) 00000100:00080000:0.0:1788575665.474518:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575665.474523:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575665.474526:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575665.474528:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575665.474532:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575665.474535:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16136 00000400:00000200:0.0:1788575665.474543:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575665.474548:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575665.474551:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575665.474553:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474556:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.474558:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474559:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575665.474562:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575665.474563:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575665.474565:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575665.474567:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575665.474569:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575665.474572:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575665.474575:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474577:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.474579:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474580:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575665.474584:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575665.474587:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474590:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575665.474592:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575665.474595:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575665.474598:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.474610:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575665.474612:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575665.474617:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575665.474619:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575665.474623:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575665.474624:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.474626:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575665.474628:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575665.474631:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575665.474737:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575665.474741:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.474748:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810452608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575665.474755:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575665.474757:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.474762:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810452608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575665.474786:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575665.474789:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575665.474792:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575665.474793:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575665.474795:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575665.474798:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575665.474800:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16136 00000400:00000200:0.0:1788575665.474804:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575665.474807:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575665.474810:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575665.474812:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474815:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.474817:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474819:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575665.474827:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575665.474828:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575665.474830:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575665.474832:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575665.474834:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575665.474836:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575665.474840:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474844:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.474848:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474853:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575665.474857:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575665.474862:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575665.474869:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575665.474874:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575665.474882:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575665.474886:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.474889:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575665.474893:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575665.474899:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.474902:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575665.474906:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575665.474910:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575665.474914:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575665.474917:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.474919:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575665.474920:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575665.474922:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575665.474927:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575665.474933:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.474938:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810452736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575665.474948:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575665.474957:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.474964:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810452736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575665.474973:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575665.474975:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575665.474978:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575665.474982:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575665.474983:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575665.474985:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575665.474987:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16136 00000400:00000200:0.0:1788575665.474994:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575665.475001:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575665.475004:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575665.475011:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475015:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475017:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475019:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575665.475023:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575665.475025:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575665.475027:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575665.475030:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575665.475036:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575665.475040:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575665.475046:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475051:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475053:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475057:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575665.475060:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575665.475067:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475079:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575665.475083:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575665.475086:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575665.475092:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475096:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575665.475098:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575665.475104:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475108:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575665.475112:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575665.475117:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575665.475123:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575665.475124:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475127:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575665.475130:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575665.475133:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575665.475137:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575665.475143:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.475150:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810452864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575665.475162:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575665.475164:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.475169:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810452864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575665.475178:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575665.475183:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575665.475193:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575665.475195:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575665.475198:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575665.475204:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575665.475208:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16136 00000400:00000200:0.0:1788575665.475215:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575665.475227:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575665.475231:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575665.475234:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475237:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475241:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475247:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575665.475253:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575665.475257:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575665.475259:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575665.475262:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575665.475266:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575665.475272:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575665.475279:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475283:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475289:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475293:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575665.475298:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575665.475303:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475308:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575665.475312:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575665.475316:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575665.475321:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475325:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575665.475329:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575665.475335:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475339:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575665.475343:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575665.475348:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575665.475354:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575665.475357:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475361:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575665.475369:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575665.475373:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575665.475378:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575665.475384:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.475393:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810452992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575665.475402:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575665.475405:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.475409:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810452992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575665.475420:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575665.475426:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575665.475433:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575665.475436:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575665.475440:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575665.475446:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575665.475452:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16136 00000400:00000200:0.0:1788575665.475460:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575665.475467:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575665.475472:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575665.475477:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475481:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475486:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475491:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575665.475495:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575665.475499:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575665.475502:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575665.475506:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575665.475511:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575665.475516:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575665.475529:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475535:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575665.475540:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475545:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575665.475549:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575665.475556:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575665.475560:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575665.475565:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575665.475570:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575665.475576:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475581:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575665.475586:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575665.475591:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475595:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575665.475600:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575665.475605:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575665.475611:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575665.475614:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575665.475617:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575665.475618:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575665.475620:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575665.475627:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575665.475634:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.475670:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810453120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575665.475697:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575665.475702:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.475709:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810453120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575665.475722:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575665.475728:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16142) 00000100:00000040:0.0:1788575669.313747:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310808287, at 4310808320 00000100:00000010:0.0:1788575669.313763:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000020:00000001:0.0:1788575669.441921:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575669.441928:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575669.441933:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575674.433748:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310813406, at 4310813440 00000100:00000010:0.0:1788575674.433761:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788575674.561826:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575674.561830:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575674.561833:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575678.722050:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575678.722056:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575678.722066:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575678.722071:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575678.722073:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575679.553789:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310818526, at 4310818560 00000100:00000010:0.0:1788575679.553802:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000020:00000001:0.0:1788575679.681857:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575679.681860:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575679.681864:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575680.834356:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575680.834366:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575680.834368:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575680.834372:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575680.834376:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575680.834384:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16147 00000400:00000200:0.0:1788575680.834394:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575680.834401:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575680.834404:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575680.834407:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834410:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834413:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834431:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575680.834435:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575680.834437:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575680.834441:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575680.834444:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575680.834446:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575680.834449:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575680.834452:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834453:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834455:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834457:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575680.834460:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575680.834464:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834467:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575680.834469:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575680.834472:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575680.834475:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834477:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575680.834479:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575680.834485:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575680.834489:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575680.834492:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575680.834493:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834496:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575680.834498:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575680.834502:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575680.834508:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575680.834515:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.834523:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810454528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575680.834533:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575680.834538:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.834549:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810454528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575680.834572:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575680.834575:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575680.834578:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575680.834580:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575680.834581:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575680.834584:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575680.834586:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16147 00000400:00000200:0.0:1788575680.834590:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575680.834594:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575680.834596:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575680.834600:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834602:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834605:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834607:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575680.834611:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575680.834613:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575680.834616:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575680.834618:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575680.834622:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575680.834627:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575680.834632:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834634:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834665:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834670:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575680.834686:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575680.834692:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834697:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575680.834706:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575680.834712:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575680.834717:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834722:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575680.834727:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575680.834734:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834737:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575680.834741:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575680.834744:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575680.834747:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575680.834749:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834753:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575680.834756:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575680.834757:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575680.834764:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575680.834772:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.834780:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810454656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575680.834791:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575680.834795:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.834800:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810454656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575680.834816:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575680.834821:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575680.834828:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575680.834831:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575680.834834:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575680.834841:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575680.834846:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16147 00000400:00000200:0.0:1788575680.834853:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575680.834859:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575680.834862:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575680.834874:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834877:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834880:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834884:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575680.834889:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575680.834890:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575680.834893:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575680.834896:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575680.834900:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575680.834903:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575680.834909:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834912:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.834916:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834919:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575680.834922:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575680.834929:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575680.834932:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575680.834935:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575680.834939:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575680.834944:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834949:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575680.834952:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575680.834957:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834959:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575680.834962:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575680.834971:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575680.834973:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575680.834976:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.834979:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575680.834982:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575680.834985:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575680.834991:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575680.835001:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.835009:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810454784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575680.835023:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575680.835027:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.835035:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810454784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575680.835050:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575680.835053:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575680.835056:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575680.835057:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575680.835059:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575680.835061:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575680.835066:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16147 00000400:00000200:0.0:1788575680.835074:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575680.835079:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575680.835082:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575680.835090:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835094:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.835099:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835102:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575680.835107:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575680.835109:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575680.835113:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575680.835115:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575680.835118:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575680.835122:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575680.835127:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835129:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.835138:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835142:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575680.835147:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575680.835152:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835156:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575680.835159:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575680.835163:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575680.835168:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835172:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575680.835175:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575680.835180:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835186:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575680.835192:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575680.835195:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575680.835198:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575680.835201:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835204:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575680.835206:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575680.835210:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575680.835217:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575680.835222:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.835229:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810454912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575680.835242:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575680.835246:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.835253:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810454912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575680.835272:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575680.835277:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575680.835283:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575680.835286:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575680.835298:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575680.835304:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575680.835309:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16147 00000400:00000200:0.0:1788575680.835318:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575680.835326:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575680.835330:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575680.835336:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835340:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.835345:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835349:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575680.835354:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575680.835357:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575680.835361:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575680.835365:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575680.835370:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575680.835376:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575680.835383:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835388:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575680.835392:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835396:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575680.835400:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575680.835404:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575680.835410:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575680.835413:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575680.835418:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575680.835423:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835426:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575680.835430:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575680.835436:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835438:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575680.835447:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575680.835451:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575680.835454:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575680.835455:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.835457:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575680.835458:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575680.835460:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575680.835464:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575680.835467:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.835476:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810455040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575680.835500:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575680.835504:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.835513:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810455040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575680.835522:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575680.835527:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16157) 00000100:00000040:0.0:1788575684.673763:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310823647, at 4310823680 00000100:00000010:0.0:1788575684.673780:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000020:00000001:0.0:1788575684.802062:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575684.802065:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575684.802068:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575685.953782:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575685.953794:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9340. 00000020:00000010:0.0:1788575685.953801:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985dc00. 00000020:00000010:0.0:1788575685.953808:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575685.953813:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985d600. 00000020:00000010:0.0:1788575685.953817:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575685.953825:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575685.953833:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575685.953841:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575685.953848:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788575685.953855:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575685.953867:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575685.953877:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575685.953887:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985d600. 00000020:00000010:0.0:1788575685.953893:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575685.953901:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4bc00. 80000000:00000010:0.0:1788575685.953907:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575685.953913:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575685.953919:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575685.953924:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985dc00. 10000000:00000010:0.0:1788575685.953930:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9340. 10000000:01000000:0.0:1788575685.953935:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575685.953942:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575685.953945:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575685.953949:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575685.953952:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575685.953955:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575685.953957:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575685.953963:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575685.953964:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575685.953967:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575685.953969:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575685.953979:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575685.953984:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575687.169760:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575687.169768:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575687.169773:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575687.169775:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575687.169777:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575687.169778:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575687.169781:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575687.169783:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575687.169785:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575687.169788:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575687.169790:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575687.169800:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575687.169804:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575687.169822:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575687.169825:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575687.169830:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575687.169837:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b600. 00000100:00000001:0.0:1788575687.169843:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575687.169847:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575687.169849:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575687.169857:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692000. 00010000:00000001:0.0:1788575687.169861:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575687.169864:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30400. 00000020:00000001:0.0:1788575687.169869:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575687.169872:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30400 with handle 0x1441586093168dad to hash 00000020:00000001:0.0:1788575687.169873:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575687.169875:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788575687.169877:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788575687.169880:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575687.169884:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575687.169888:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575687.169896:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575687.169907:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575687.169913:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575687.169915:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575687.169921:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575687.169923:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575687.169925:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.169927:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575687.169931:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575687.169935:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.169941:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575687.169943:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.169952:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.169968:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575687.169975:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.169982:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575687.169986:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.169992:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575687.169995:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575687.169997:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.169999:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575687.170002:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575687.170004:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575687.170007:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.170009:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.170011:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.170015:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.170022:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810455808:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575687.170027:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.170029:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575687.170031:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575687.170032:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.170035:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.170041:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575687.170043:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575687.170053:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575687.170055:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575687.170057:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575687.170060:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575687.170062:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575687.170064:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=4 00000020:00000001:0.0:1788575687.170066:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788575687.170068:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00010000:0.0:1788575687.170071:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170077:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575687.170081:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170086:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575687.170089:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170094:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575687.170096:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575687.170098:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170103:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575687.170105:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575687.170108:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575687.170110:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575687.170116:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30400 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575687.170119:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575687.170126:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170139:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575687.170141:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575687.170143:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575687.170144:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30400 refcount=5 00000020:00000001:0.0:1788575687.170146:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788575687.170148:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681984 : -109714172869632 : ffff9c372db30400) 00010000:00000001:0.0:1788575687.170152:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575687.170154:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575687.170160:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575687.170163:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575687.170169:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575687.170172:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575687.170173:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575687.170176:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575687.170178:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575687.170180:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575687.170182:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575687.170183:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575687.170185:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575687.170188:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30400 with handle 0x1441586093168dad from hash 00010000:00000001:0.0:1788575687.170190:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575687.170191:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575687.170192:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575687.170193:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575687.170195:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575687.170196:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575687.170198:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575687.170199:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575687.170209:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575687.170211:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575687.170215:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575687.170220:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575687.170222:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575687.170223:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575687.170225:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575687.170226:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575687.170227:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575687.170228:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575687.170230:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30400/0x1441586093168dad lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575687.170238:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692000 count: 0 00010000:00000001:0.0:1788575687.170246:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575687.170248:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575687.170251:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.170255:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810455808/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.170262:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575687.170264:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.170266:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575687.170268:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.170271:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b600. 02000000:00000001:0.0:1788575687.170275:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.170277:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.170278:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575687.170281:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.170282:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575687.170284:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575687.170286:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575687.170289:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575689.921798:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575689.921801:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575689.921818:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575691.009778:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310829984, at 4310830016 00000100:00000010:0.0:1788575691.009796:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000020:00000001:0.0:1788575695.041845:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575695.041848:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575695.041851:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575696.449777:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310835422, at 4310835456 00000100:00000010:0.0:1788575696.449792:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000020:00000001:0.0:1788575700.161884:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575700.161888:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575700.161891:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575701.569757:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310840542, at 4310840576 00000100:00000010:0.0:1788575701.569773:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb660. 00000020:00000001:0.0:1788575705.281916:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575705.281923:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575705.281928:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575706.689776:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310845662, at 4310845696 00000100:00000010:0.0:1788575706.689792:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 10000000:00000001:0.0:1788575707.457802:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575707.457815:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9d80. 00000020:00000010:0.0:1788575707.457822:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575707.457827:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575707.457831:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575707.457836:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575707.457840:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575707.457844:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575707.457848:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575707.457855:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985dc00. 10000000:00000040:0.0:1788575707.457859:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575707.457862:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575707.457866:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575707.457873:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575707.457888:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575707.457892:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575707.457894:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575707.457897:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575707.457900:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985dc00. 00000020:00000010:0.0:1788575707.457903:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4b600. 10000000:00000010:0.0:1788575707.457905:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9d80. 10000000:01000000:0.0:1788575707.457907:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575707.457910:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575707.457913:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575707.457914:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575707.457916:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575707.457919:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575707.457920:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575707.457923:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575707.457925:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575707.457927:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575707.457929:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575707.457931:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575707.457933:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575709.441756:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575709.441762:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575709.441765:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575709.441767:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575709.441769:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575709.441770:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575709.441772:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575709.441774:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575709.441776:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575709.441779:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575709.441780:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575709.441789:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575709.441793:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575709.441797:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575709.441799:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575709.441803:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575709.441808:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788575709.441825:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575709.441828:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575709.441830:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575709.441836:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692800. 00010000:00000001:0.0:1788575709.441839:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575709.441842:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30a00. 00000020:00000001:0.0:1788575709.441846:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575709.441848:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30a00 with handle 0x1441586093168db4 to hash 00000020:00000001:0.0:1788575709.441850:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575709.441851:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575709.441853:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575709.441856:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575709.441858:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575709.441861:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575709.441868:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575709.441877:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575709.441881:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575709.441882:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575709.441886:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575709.441888:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575709.441890:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575709.441891:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575709.441895:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575709.441898:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575709.441900:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575709.441901:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575709.441908:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575709.442179:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575709.442183:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575709.442188:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575709.442190:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575709.442195:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575709.442196:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575709.442197:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575709.442199:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575709.442200:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575709.442202:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575709.442205:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.442206:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575709.442208:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575709.442210:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575709.442215:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810459136:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575709.442219:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.442220:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575709.442222:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575709.442223:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575709.442225:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575709.442229:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575709.442231:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575709.442235:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575709.442236:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575709.442238:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575709.442244:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575709.442245:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575709.442247:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=4 00000020:00000001:0.0:1788575709.442248:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575709.442250:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00010000:0.0:1788575709.442253:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442257:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575709.442259:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442263:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575709.442265:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442269:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575709.442270:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575709.442271:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442274:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575709.442275:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575709.442277:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575709.442279:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575709.442282:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575709.442284:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575709.442285:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442293:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575709.442294:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575709.442295:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575709.442296:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30a00 refcount=5 00000020:00000001:0.0:1788575709.442297:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575709.442298:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683520 : -109714172868096 : ffff9c372db30a00) 00010000:00000001:0.0:1788575709.442300:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575709.442302:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575709.442305:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575709.442306:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575709.442309:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575709.442310:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575709.442311:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575709.442313:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575709.442314:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575709.442316:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575709.442317:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575709.442318:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575709.442319:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575709.442320:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30a00 with handle 0x1441586093168db4 from hash 00010000:00000001:0.0:1788575709.442322:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575709.442322:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575709.442323:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575709.442324:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575709.442324:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575709.442325:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575709.442327:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575709.442327:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575709.442328:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575709.442331:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575709.442333:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575709.442337:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575709.442337:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575709.442338:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575709.442339:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575709.442340:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575709.442341:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575709.442341:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575709.442342:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30a00/0x1441586093168db4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575709.442346:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692800 count: 0 00010000:00000001:0.0:1788575709.442352:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575709.442354:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575709.442356:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575709.442360:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810459136/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575709.442366:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575709.442368:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575709.442370:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575709.442371:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575709.442373:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788575709.442377:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575709.442379:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575709.442381:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575709.442384:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575709.442386:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575709.442387:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575709.442390:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575709.442393:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000020:00000001:0.0:1788575710.401829:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575710.401833:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575710.401837:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575713.281769:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310852255, at 4310852288 00000100:00000010:0.0:1788575713.281787:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788575714.626167:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575714.626179:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575714.626182:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575714.626185:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575714.626190:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575714.626197:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16181 00000400:00000200:0.0:1788575714.626206:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575714.626211:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575714.626215:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575714.626218:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626221:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626223:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626225:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575714.626229:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575714.626231:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575714.626234:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575714.626237:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575714.626239:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575714.626241:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575714.626244:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626246:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626247:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626249:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575714.626252:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575714.626255:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626257:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575714.626260:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575714.626264:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575714.626283:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626285:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575714.626288:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575714.626293:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575714.626297:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575714.626301:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575714.626302:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626305:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575714.626307:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575714.626311:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575714.626318:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575714.626324:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.626333:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810459904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575714.626342:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575714.626347:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626351:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810459904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575714.626382:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575714.626385:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575714.626389:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575714.626390:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575714.626392:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575714.626394:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575714.626397:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16181 00000400:00000200:0.0:1788575714.626401:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575714.626404:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575714.626406:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575714.626409:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626411:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626413:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626421:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575714.626423:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575714.626424:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575714.626426:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575714.626428:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575714.626429:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575714.626431:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575714.626434:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626436:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626438:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626440:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575714.626442:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575714.626444:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626447:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575714.626449:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575714.626454:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575714.626456:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626458:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575714.626462:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575714.626465:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626467:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575714.626470:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575714.626473:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575714.626475:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575714.626477:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626479:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575714.626480:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575714.626482:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575714.626485:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575714.626489:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.626494:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810460032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575714.626511:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575714.626513:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626517:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810460032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575714.626525:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575714.626527:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575714.626530:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575714.626532:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575714.626533:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575714.626535:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575714.626538:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16181 00000400:00000200:0.0:1788575714.626542:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575714.626544:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575714.626547:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575714.626549:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626551:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626553:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626555:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575714.626557:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575714.626558:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575714.626559:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575714.626561:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575714.626563:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575714.626565:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575714.626568:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626569:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626571:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626573:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575714.626581:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575714.626583:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626586:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575714.626587:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575714.626590:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575714.626591:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626593:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575714.626595:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575714.626597:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626599:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575714.626602:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575714.626604:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575714.626606:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575714.626607:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626609:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575714.626610:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575714.626612:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575714.626615:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575714.626618:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.626623:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810460160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575714.626630:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575714.626632:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626678:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810460160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575714.626686:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575714.626689:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575714.626691:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575714.626693:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575714.626694:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575714.626697:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575714.626704:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16181 00000400:00000200:0.0:1788575714.626708:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575714.626711:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575714.626713:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575714.626716:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626718:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626720:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626721:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575714.626723:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575714.626725:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575714.626726:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575714.626728:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575714.626730:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575714.626731:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575714.626734:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626735:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626737:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626738:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575714.626741:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575714.626743:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626745:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575714.626746:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575714.626749:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575714.626751:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626753:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575714.626755:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575714.626757:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626759:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575714.626763:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575714.626765:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575714.626767:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575714.626772:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626774:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575714.626775:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575714.626777:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575714.626780:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575714.626784:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.626788:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810460288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575714.626794:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575714.626796:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626799:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810460288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575714.626806:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575714.626808:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575714.626811:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575714.626812:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575714.626813:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575714.626816:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575714.626818:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16181 00000400:00000200:0.0:1788575714.626821:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575714.626824:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575714.626826:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575714.626828:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626830:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626832:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626834:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575714.626836:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575714.626837:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575714.626838:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575714.626840:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575714.626842:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575714.626848:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575714.626851:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626853:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575714.626855:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626856:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575714.626859:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575714.626860:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575714.626862:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575714.626864:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575714.626866:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575714.626868:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626870:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575714.626872:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575714.626874:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626875:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575714.626878:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575714.626880:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575714.626882:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575714.626883:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.626885:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575714.626887:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575714.626888:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575714.626891:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575714.626895:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.626899:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810460416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575714.626906:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575714.626908:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626911:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810460416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575714.626918:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575714.626926:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16191) 00000020:00000001:0.0:1788575715.522355:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575715.522358:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575715.522360:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788575716.610271:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575718.465766:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310857439, at 4310857472 00000100:00000010:0.0:1788575718.465783:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000020:00000001:0.0:1788575720.641863:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575720.641866:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575720.641868:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788575722.817933:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788575722.817976:0:2392:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788575722.817986:0:2392:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575722.817990:0:2392:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788575722.817994:0:2392:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575722.817998:0:2392:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788575722.818004:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575722.818007:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fd00. 00000400:00000200:0.0:1788575722.818012:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575722.818017:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575722.818022:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf12a8 00000400:00000010:0.0:1788575722.818025:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf12a8. 00000100:00000001:0.0:1788575722.818030:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575722.818038:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c5f80 x1875440810460544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818063:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575722.818066:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810460544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818073:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.818075:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575722.818082:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575722.818084:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f300. 00000400:00000200:0.0:1788575722.818098:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575722.818101:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575722.818104:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1110 00000400:00000010:0.0:1788575722.818106:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1110. 00000100:00000001:0.0:1788575722.818108:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575722.818112:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c7800 x1875440810460672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818120:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575722.818123:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440810460672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818129:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.818130:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575722.818135:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575722.818137:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f800. 00000400:00000200:0.0:1788575722.818140:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575722.818148:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575722.818151:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf16e8 00000400:00000010:0.0:1788575722.818153:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf16e8. 00000100:00000001:0.0:1788575722.818155:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575722.818159:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c4e00 x1875440810460800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818167:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575722.818170:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4e00 x1875440810460800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818177:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.818179:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575722.818185:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575722.818189:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696f400. 00000400:00000200:0.0:1788575722.818195:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575722.818201:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575722.818207:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1a18 00000400:00000010:0.0:1788575722.818218:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1a18. 00000100:00000001:0.0:1788575722.818224:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575722.818230:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c7480 x1875440810460928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818243:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575722.818248:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7480 x1875440810460928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818260:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.818264:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575722.818275:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788575722.818281:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c370696fa00. 00000400:00000200:0.0:1788575722.818287:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575722.818292:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575722.818295:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c371edf1440 00000400:00000010:0.0:1788575722.818298:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c371edf1440. 00000100:00000001:0.0:1788575722.818303:0:2392:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575722.818311:0:2392:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371f1c6300 x1875440810461056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818340:0:2392:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575722.818348:0:2392:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440810461056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575744 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575722.818369:0:2392:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.818373:0:2392:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788575723.585713:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310862558, at 4310862592 00000100:00000010:0.0:1788575723.585726:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbf00. 00000020:00000001:0.0:1788575725.762107:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575725.762120:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575725.762124:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575728.705747:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310867679, at 4310867712 00000100:00000010:0.0:1788575728.705769:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb7e0. 10000000:00000001:0.0:1788575729.473775:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575729.473808:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9e80. 00000020:00000010:0.0:1788575729.473816:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985dc00. 00000020:00000010:0.0:1788575729.473821:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575729.473824:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985d600. 00000020:00000010:0.0:1788575729.473829:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575729.473837:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575729.473841:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575729.473845:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575729.473849:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788575729.473853:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575729.473855:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575729.473860:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788575729.473868:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985d600. 00000020:00000010:0.0:1788575729.473870:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575729.473874:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788575729.473876:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575729.473879:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575729.473881:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788575729.473883:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985dc00. 10000000:00000010:0.0:1788575729.473885:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9e80. 10000000:01000000:0.0:1788575729.473888:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575729.473890:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575729.473892:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575729.473894:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575729.473895:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575729.473897:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575729.473899:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575729.473902:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575729.473903:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575729.473905:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575729.473908:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575729.473909:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575729.473911:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788575729.985752:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575729.985789:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575729.985792:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575729.985798:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575729.985805:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575729.985814:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16196 00000400:00000200:0.0:1788575729.985826:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575729.985834:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575729.985844:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575729.985847:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.985851:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.985853:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.985857:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575729.985860:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575729.985862:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575729.985865:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575729.985868:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575729.985870:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575729.985871:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575729.985873:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.985875:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.985876:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.985877:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575729.985879:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575729.985881:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575729.985883:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575729.985885:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575729.985889:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575729.985891:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.985894:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575729.985896:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575729.985904:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575729.985914:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575729.985918:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575729.985920:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.985924:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575729.985927:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575729.985933:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575729.985942:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575729.985950:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.985959:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810461824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575729.985967:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575729.986143:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.986150:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810461824/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575729.986176:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575729.986180:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575729.986184:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575729.986185:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575729.986187:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575729.986190:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575729.986193:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16196 00000400:00000200:0.0:1788575729.986197:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575729.986200:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575729.986204:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575729.986208:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986210:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986213:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986216:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575729.986219:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575729.986221:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575729.986223:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575729.986232:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575729.986236:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575729.986242:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575729.986250:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986254:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986258:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986264:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575729.986268:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575729.986275:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986281:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575729.986288:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575729.986297:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575729.986302:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986307:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575729.986316:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575729.986325:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986328:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575729.986332:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575729.986335:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575729.986338:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575729.986339:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986342:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575729.986346:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575729.986352:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575729.986362:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575729.986371:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.986382:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810461952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575729.986400:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575729.986404:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.986410:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810461952/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575729.986451:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575729.986458:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575729.986462:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575729.986464:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575729.986468:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575729.986473:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575729.986479:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16196 00000400:00000200:0.0:1788575729.986495:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575729.986511:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575729.986516:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575729.986523:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986533:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986541:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986546:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575729.986552:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575729.986553:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575729.986558:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575729.986562:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575729.986565:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575729.986569:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575729.986577:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986583:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986591:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986597:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575729.986603:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575729.986613:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986619:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575729.986629:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575729.986685:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575729.986690:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986697:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575729.986703:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575729.986714:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986722:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575729.986732:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575729.986736:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575729.986738:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575729.986739:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986741:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575729.986743:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575729.986744:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575729.986748:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575729.986751:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.986756:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810462080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575729.986762:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575729.986764:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.986776:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810462080/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575729.986786:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575729.986794:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575729.986798:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575729.986801:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575729.986803:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575729.986812:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575729.986816:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16196 00000400:00000200:0.0:1788575729.986821:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575729.986825:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575729.986831:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575729.986842:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986846:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986851:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986856:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575729.986860:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575729.986866:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575729.986868:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575729.986870:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575729.986873:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575729.986880:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575729.986886:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986890:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.986896:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986900:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575729.986903:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575729.986908:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575729.986910:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575729.986912:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575729.986916:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575729.986919:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986922:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575729.986927:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575729.986930:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986934:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575729.986937:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575729.986942:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575729.986946:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575729.986948:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.986951:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575729.986954:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575729.986957:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575729.986966:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575729.986986:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.986998:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810462208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575729.987022:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575729.987030:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.987039:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810462208/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575729.987059:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575729.987062:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575729.987070:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575729.987073:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575729.987076:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575729.987084:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575729.987088:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16196 00000400:00000200:0.0:1788575729.987097:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575729.987100:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575729.987102:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575729.987106:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.987108:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.987111:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.987113:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575729.987121:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575729.987122:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575729.987124:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575729.987127:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575729.987129:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575729.987132:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575729.987143:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.987146:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575729.987163:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575729.987169:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575729.987180:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575729.987188:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575729.987196:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575729.987201:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575729.987205:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575729.987207:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.987209:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575729.987211:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575729.987213:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.987214:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575729.987217:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575729.987220:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575729.987223:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575729.987225:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575729.987232:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575729.987235:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575729.987239:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575729.987243:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575729.987251:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.987256:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810462336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575729.987269:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575729.987274:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.987284:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810462336/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575729.987307:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575729.987313:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16206) 00000020:00000001:0.0:1788575730.881831:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575730.881834:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575730.881836:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788575731.457734:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575731.457741:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575731.457746:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575731.457748:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575731.457750:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575731.457751:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575731.457755:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575731.457756:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575731.457759:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575731.457761:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575731.457763:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575731.457773:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575731.457778:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575731.457782:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575731.457786:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575731.457791:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575731.457797:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788575731.457804:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575731.457809:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575731.457811:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575731.457818:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692100. 00010000:00000001:0.0:1788575731.457822:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575731.457825:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31200. 00000020:00000001:0.0:1788575731.457828:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575731.457831:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31200 with handle 0x1441586093168dbb to hash 00000020:00000001:0.0:1788575731.457833:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575731.457835:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575731.457838:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575731.457841:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575731.457845:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575731.457849:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575731.457859:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575731.457888:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575731.457894:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575731.457919:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575731.457926:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575731.457928:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575731.457931:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575731.457932:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788575731.457937:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575731.457941:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575731.457943:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575731.457945:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575731.457954:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575731.457964:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575731.457968:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575731.457982:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575731.457989:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575731.457996:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575731.457998:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575731.458000:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575731.458003:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575731.458005:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575731.458007:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575731.458010:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575731.458013:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575731.458015:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575731.458024:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575731.458032:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810462464:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575731.458037:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575731.458040:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575731.458043:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575731.458044:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575731.458048:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575731.458055:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575731.458058:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575731.458063:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575731.458065:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575731.458068:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575731.458071:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575731.458073:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575731.458075:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=4 00000020:00000001:0.0:1788575731.458077:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575731.458080:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00010000:0.0:1788575731.458084:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458090:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575731.458095:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458100:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575731.458103:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458116:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575731.458118:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575731.458120:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458126:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575731.458128:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575731.458130:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575731.458133:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575731.458139:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575731.458142:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575731.458144:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458150:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575731.458151:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575731.458152:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575731.458154:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31200 refcount=5 00000020:00000001:0.0:1788575731.458155:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575731.458157:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685568 : -109714172866048 : ffff9c372db31200) 00010000:00000001:0.0:1788575731.458159:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575731.458162:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575731.458168:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575731.458171:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575731.458176:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575731.458182:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575731.458183:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575731.458186:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575731.458188:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575731.458190:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575731.458192:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575731.458193:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575731.458195:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575731.458198:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31200 with handle 0x1441586093168dbb from hash 00010000:00000001:0.0:1788575731.458200:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575731.458202:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575731.458203:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575731.458204:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575731.458205:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575731.458207:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575731.458209:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575731.458210:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575731.458212:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575731.458213:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575731.458216:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575731.458221:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575731.458222:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575731.458223:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575731.458224:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575731.458226:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575731.458227:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575731.458228:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575731.458230:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31200/0x1441586093168dbb lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575731.458236:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692100 count: 0 00010000:00000001:0.0:1788575731.458244:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575731.458246:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575731.458248:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575731.458255:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810462464/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575731.458262:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575731.458264:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575731.458266:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575731.458268:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575731.458270:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788575731.458274:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575731.458276:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575731.458277:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788575731.458281:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575731.458282:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575731.458284:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575731.458285:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575731.458288:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575735.297763:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310874271, at 4310874304 00000100:00000010:0.0:1788575735.297785:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb9c0. 00000020:00000001:0.0:1788575736.001855:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575736.001861:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575736.001866:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575736.643008:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575736.643018:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575736.643021:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575736.643025:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575736.643029:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575736.643037:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16203 00000400:00000200:0.0:1788575736.643046:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575736.643052:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575736.643055:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575736.643058:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643061:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643063:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643066:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575736.643087:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575736.643089:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575736.643092:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575736.643096:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575736.643098:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575736.643100:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575736.643104:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643106:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643107:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643109:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575736.643112:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575736.643115:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643118:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575736.643121:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575736.643124:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575736.643127:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643129:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575736.643132:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575736.643137:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575736.643140:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575736.643144:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575736.643146:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643149:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575736.643151:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575736.643155:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575736.643162:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575736.643168:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.643177:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810463232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575736.643186:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575736.643190:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.643194:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810463232/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575736.643220:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575736.643223:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575736.643226:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575736.643227:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575736.643229:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575736.643231:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575736.643234:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16203 00000400:00000200:0.0:1788575736.643238:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575736.643241:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575736.643244:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575736.643248:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643250:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643252:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643254:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575736.643257:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575736.643259:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575736.643262:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575736.643265:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575736.643269:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575736.643273:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575736.643279:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643283:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643287:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643291:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575736.643296:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575736.643301:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643305:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575736.643312:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575736.643317:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575736.643321:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643325:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575736.643331:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575736.643336:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643340:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575736.643346:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575736.643350:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575736.643354:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575736.643357:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643361:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575736.643364:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575736.643366:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575736.643372:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575736.643377:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.643386:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810463360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575736.643396:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575736.643399:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.643403:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810463360/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575736.643419:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575736.643424:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575736.643431:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575736.643434:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575736.643437:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575736.643441:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575736.643447:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16203 00000400:00000200:0.0:1788575736.643454:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575736.643458:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575736.643460:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575736.643468:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643470:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643473:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643477:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575736.643480:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575736.643483:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575736.643487:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575736.643490:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575736.643492:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575736.643497:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575736.643503:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643505:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643511:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643515:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575736.643518:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575736.643525:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643530:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575736.643534:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575736.643539:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575736.643544:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643550:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575736.643553:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575736.643559:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643564:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575736.643568:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575736.643572:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575736.643578:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575736.643579:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643581:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575736.643583:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575736.643584:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575736.643588:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575736.643601:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.643608:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810463488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575736.643620:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575736.643625:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.643632:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810463488/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575736.643702:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575736.643706:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575736.643712:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575736.643717:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575736.643719:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575736.643724:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575736.643730:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16203 00000400:00000200:0.0:1788575736.643741:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575736.643749:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575736.643754:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575736.643763:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643769:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643775:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643781:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575736.643787:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575736.643790:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575736.643795:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575736.643798:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575736.643802:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575736.643807:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575736.643811:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643815:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.643825:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643831:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575736.643835:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575736.643839:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575736.643841:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575736.643843:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575736.643847:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575736.643851:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643857:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575736.643860:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575736.643864:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643866:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575736.643872:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575736.643876:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575736.643882:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575736.643884:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.643886:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575736.643888:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575736.643891:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575736.643897:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575736.643904:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.643913:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810463616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575736.643929:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575736.643933:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.643940:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810463616/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575736.643958:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575736.643963:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575736.643970:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575736.643973:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575736.643982:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575736.643988:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575736.643994:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16203 00000400:00000200:0.0:1788575736.644002:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575736.644011:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575736.644017:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575736.644023:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.644028:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.644032:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.644037:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575736.644041:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575736.644043:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575736.644047:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575736.644050:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575736.644055:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575736.644059:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575736.644065:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.644069:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575736.644074:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575736.644077:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575736.644082:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575736.644086:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575736.644089:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575736.644092:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575736.644097:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575736.644100:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.644102:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575736.644104:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575736.644109:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.644113:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575736.644125:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575736.644130:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575736.644136:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575736.644139:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.644144:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575736.644147:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575736.644150:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575736.644157:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575736.644163:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.644172:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810463744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575736.644187:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575736.644191:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.644199:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810463744/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575736.644216:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575736.644222:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16213) 00000100:00000040:0.0:1788575740.481741:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310879456, at 4310879488 00000100:00000010:0.0:1788575740.481757:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbbd20. 00000020:00000001:0.0:1788575741.121879:0:237083:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575741.121884:0:237083:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575741.121888:0:237083:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575745.602371:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310884575, at 4310884608 00000100:00000010:0.0:1788575745.602388:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00080000:0.0:1788575746.881805:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575746.881817:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575746.881819:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575746.881823:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575746.881828:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575746.881836:0:234251:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16213 00000400:00000200:0.0:1788575746.881844:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575746.881879:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575746.881882:0:234251:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575746.881886:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.881889:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.881891:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.881893:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575746.881897:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575746.881899:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575746.881901:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575746.881905:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575746.881907:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575746.881909:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575746.881912:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.881914:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.881915:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.881917:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575746.881920:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575746.881923:0:234251:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575746.881925:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575746.881928:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575746.881932:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575746.881934:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.881937:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575746.881939:0:234251:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575746.881946:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575746.881949:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575746.881953:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575746.881955:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.881958:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575746.881961:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575746.881964:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575746.881972:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575746.881984:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.881994:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810464512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575746.882003:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575746.882008:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.882012:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810464512/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575746.882039:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575746.882042:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575746.882046:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575746.882047:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575746.882048:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575746.882051:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575746.882054:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16213 00000400:00000200:0.0:1788575746.882058:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575746.882061:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575746.882064:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575746.882067:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882070:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882072:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882074:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575746.882077:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575746.882078:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575746.882080:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575746.882082:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575746.882084:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575746.882086:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575746.882089:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882092:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882102:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882106:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575746.882111:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575746.882118:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882125:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575746.882129:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575746.882136:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575746.882141:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882157:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575746.882163:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575746.882170:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882174:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575746.882181:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575746.882185:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575746.882189:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575746.882193:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882197:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575746.882198:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575746.882201:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575746.882209:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575746.882215:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.882223:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810464640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575746.882235:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575746.882239:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.882246:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810464640/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575746.882263:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575746.882268:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575746.882275:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575746.882279:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575746.882289:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575746.882296:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575746.882301:0:234251:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16213 00000400:00000200:0.0:1788575746.882352:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575746.882361:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575746.882365:0:234251:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575746.882371:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882375:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882378:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882379:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575746.882382:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575746.882383:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575746.882386:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575746.882390:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575746.882394:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575746.882398:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575746.882405:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882409:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882412:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882417:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575746.882422:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575746.882429:0:234251:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882433:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575746.882436:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575746.882442:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575746.882447:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882452:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575746.882455:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575746.882460:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882463:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575746.882473:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575746.882478:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575746.882484:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575746.882487:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882491:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575746.882494:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575746.882498:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575746.882504:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575746.882510:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.882519:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810464768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575746.882535:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575746.882539:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.882547:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810464768/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575746.882558:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575746.882561:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575746.882567:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575746.882569:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575746.882572:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575746.882575:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575746.882580:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16213 00000400:00000200:0.0:1788575746.882590:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575746.882596:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575746.882600:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575746.882608:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882613:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882618:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882621:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575746.882626:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575746.882672:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575746.882676:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575746.882681:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575746.882687:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575746.882692:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575746.882701:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882706:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882711:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882717:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575746.882723:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575746.882729:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882734:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575746.882738:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575746.882743:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575746.882747:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882753:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575746.882757:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575746.882762:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882766:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575746.882771:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575746.882776:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575746.882782:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575746.882785:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.882788:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575746.882792:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575746.882796:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575746.882802:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575746.882809:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.882819:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810464896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575746.882828:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575746.882829:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.882839:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810464896/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575746.882853:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575746.882858:0:234251:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575746.882866:0:234251:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575746.882869:0:234251:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575746.882873:0:234251:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575746.882878:0:234251:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575746.882884:0:234251:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16213 00000400:00000200:0.0:1788575746.882894:0:234251:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575746.882903:0:234251:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575746.882907:0:234251:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575746.882912:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882917:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882923:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882929:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575746.882934:0:234251:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575746.882937:0:234251:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575746.882942:0:234251:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575746.882945:0:234251:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575746.882948:0:234251:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575746.882953:0:234251:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575746.882960:0:234251:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882964:0:234251:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575746.882970:0:234251:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882973:0:234251:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575746.882976:0:234251:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575746.882982:0:234251:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575746.882987:0:234251:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575746.883002:0:234251:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575746.883007:0:234251:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575746.883014:0:234251:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.883019:0:234251:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575746.883023:0:234251:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575746.883030:0:234251:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.883034:0:234251:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575746.883042:0:234251:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575746.883047:0:234251:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575746.883054:0:234251:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575746.883057:0:234251:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.883059:0:234251:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575746.883063:0:234251:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575746.883068:0:234251:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575746.883075:0:234251:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575746.883082:0:234251:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.883090:0:234251:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810465024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575746.883104:0:234251:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575746.883108:0:234251:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.883119:0:234251:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810465024/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575746.883137:0:234251:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575746.883160:0:234251:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16223) 00000100:00000040:0.0:1788575750.721735:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310889694, at 4310889728 00000100:00000010:0.0:1788575750.721752:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000800:00000200:0.0:1788575754.498040:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575756.609893:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310895583, at 4310895616 00000100:00000010:0.0:1788575756.609902:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00000040:0.0:1788575761.985765:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310900958, at 4310900992 00000100:00000010:0.0:1788575761.985773:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00080000:0.0:1788575763.265759:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575763.265780:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575763.265782:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575763.265784:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575763.265788:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575763.265793:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16229 00000400:00000200:0.0:1788575763.265799:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575763.265804:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575763.265806:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575763.265808:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265810:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.265812:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265814:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575763.265816:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575763.265818:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575763.265820:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575763.265822:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575763.265823:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575763.265824:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575763.265826:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265827:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.265829:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265830:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575763.265832:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575763.265834:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265835:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575763.265837:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575763.265839:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575763.265841:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.265842:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575763.265844:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575763.265848:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575763.265854:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575763.265857:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575763.265858:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.265861:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575763.265863:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575763.265865:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575763.265902:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575763.265907:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.265913:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810467200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575763.265920:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575763.265923:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.265926:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810467200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575763.265940:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575763.265942:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575763.265944:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575763.265945:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575763.265946:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575763.265948:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575763.265949:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16229 00000400:00000200:0.0:1788575763.265952:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575763.265954:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575763.265955:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575763.265957:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265958:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.265960:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265961:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575763.265963:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575763.265963:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575763.265964:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575763.265970:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575763.265971:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575763.265973:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575763.265975:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265977:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.265978:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265979:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575763.265981:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575763.265985:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575763.265988:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575763.265991:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575763.266007:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575763.266011:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266013:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575763.266017:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575763.266019:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266020:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575763.266023:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575763.266024:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575763.266027:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575763.266029:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266031:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575763.266032:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575763.266034:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575763.266039:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575763.266051:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.266059:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810467328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575763.266072:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575763.266077:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.266082:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810467328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575763.266111:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575763.266115:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575763.266122:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575763.266125:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575763.266127:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575763.266131:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575763.266135:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16229 00000400:00000200:0.0:1788575763.266139:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575763.266144:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575763.266148:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575763.266152:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266155:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266159:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266163:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575763.266166:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575763.266169:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575763.266171:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575763.266175:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575763.266176:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575763.266181:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575763.266184:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266186:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266189:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266193:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575763.266195:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575763.266196:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266198:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575763.266204:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575763.266209:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575763.266214:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266218:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575763.266223:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575763.266228:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266232:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575763.266238:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575763.266244:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575763.266250:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575763.266252:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266255:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575763.266258:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575763.266261:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575763.266265:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575763.266269:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.266275:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810467456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575763.266284:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575763.266285:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.266290:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810467456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575763.266297:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575763.266301:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575763.266304:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575763.266307:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575763.266308:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575763.266315:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575763.266318:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16229 00000400:00000200:0.0:1788575763.266325:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575763.266330:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575763.266332:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575763.266338:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266339:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266340:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266341:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575763.266353:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575763.266354:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575763.266355:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575763.266357:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575763.266358:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575763.266360:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575763.266364:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266366:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266369:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266373:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575763.266378:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575763.266384:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266389:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575763.266393:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575763.266396:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575763.266401:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266405:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575763.266408:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575763.266414:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266417:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575763.266421:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575763.266425:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575763.266431:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575763.266434:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266436:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575763.266439:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575763.266442:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575763.266446:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575763.266454:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.266462:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810467584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575763.266469:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575763.266472:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.266475:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810467584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575763.266482:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575763.266485:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575763.266490:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575763.266492:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575763.266494:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575763.266497:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575763.266501:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16229 00000400:00000200:0.0:1788575763.266505:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575763.266510:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575763.266514:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575763.266516:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266519:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266524:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266527:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575763.266530:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575763.266531:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575763.266532:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575763.266533:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575763.266534:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575763.266535:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575763.266537:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266538:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575763.266544:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266547:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575763.266551:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575763.266554:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575763.266568:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575763.266572:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575763.266577:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575763.266580:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266586:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575763.266589:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575763.266594:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266597:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575763.266604:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575763.266609:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575763.266613:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575763.266617:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575763.266619:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575763.266621:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575763.266623:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575763.266629:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575763.266633:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.266681:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810467712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575763.266689:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575763.266692:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.266698:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810467712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575763.266715:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.266719:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16239) 00000100:00000040:0.0:1788575767.105740:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310906078, at 4310906112 00000100:00000010:0.0:1788575767.105759:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00080000:0.0:1788575768.385699:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575768.385724:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575768.385726:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575768.385730:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575768.385734:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575768.385741:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16234 00000400:00000200:0.0:1788575768.385748:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575768.385754:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575768.385757:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575768.385760:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385763:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.385765:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385768:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575768.385772:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575768.385774:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575768.385777:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575768.385780:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575768.385782:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575768.385785:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575768.385787:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385789:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.385791:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385793:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575768.385796:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575768.385800:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385802:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575768.385805:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575768.385809:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575768.385812:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.385814:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575768.385816:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575768.385823:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575768.385831:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575768.385834:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575768.385836:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.385838:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575768.385841:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575768.385845:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575768.385852:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575768.385859:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.385868:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810467840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575768.385878:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575768.385882:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.385887:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810467840/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575768.385913:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575768.385917:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575768.385920:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575768.385921:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575768.385923:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575768.385926:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575768.385929:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16234 00000400:00000200:0.0:1788575768.385933:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575768.385938:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575768.385941:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575768.385944:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385949:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.385955:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.385960:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575768.385964:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575768.385967:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575768.385971:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575768.385983:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575768.385988:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575768.385993:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575768.386000:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386005:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386009:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386011:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575768.386014:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575768.386020:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386024:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575768.386028:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575768.386044:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575768.386049:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386053:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575768.386058:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575768.386066:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386071:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575768.386077:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575768.386083:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575768.386090:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575768.386106:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386109:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575768.386111:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575768.386115:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575768.386121:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575768.386127:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.386137:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810467968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575768.386157:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575768.386169:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.386180:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810467968/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575768.386219:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575768.386223:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575768.386228:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575768.386229:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575768.386232:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575768.386237:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575768.386242:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16234 00000400:00000200:0.0:1788575768.386251:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575768.386259:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575768.386263:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575768.386268:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386272:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386274:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386276:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575768.386279:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575768.386282:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575768.386286:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575768.386289:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575768.386294:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575768.386297:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575768.386303:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386306:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386310:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386312:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575768.386317:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575768.386322:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386327:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575768.386335:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575768.386340:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575768.386345:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386349:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575768.386353:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575768.386359:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386365:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575768.386371:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575768.386376:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575768.386380:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575768.386383:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386388:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575768.386390:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575768.386393:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575768.386399:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575768.386404:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.386411:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810468096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575768.386424:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575768.386428:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.386436:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810468096/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575768.386452:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575768.386457:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575768.386463:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575768.386467:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575768.386470:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575768.386476:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575768.386481:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16234 00000400:00000200:0.0:1788575768.386489:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575768.386495:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575768.386497:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575768.386507:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386509:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386511:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386513:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575768.386515:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575768.386516:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575768.386517:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575768.386520:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575768.386525:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575768.386529:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575768.386536:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386541:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386547:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386550:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575768.386562:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575768.386568:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386574:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575768.386578:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575768.386582:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575768.386585:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386588:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575768.386591:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575768.386594:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386596:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575768.386600:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575768.386604:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575768.386607:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575768.386609:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386612:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575768.386614:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575768.386616:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575768.386620:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575768.386628:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.386651:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810468224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575768.386659:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575768.386661:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.386666:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810468224/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575768.386676:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575768.386680:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575768.386686:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575768.386689:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575768.386692:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575768.386697:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575768.386702:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16234 00000400:00000200:0.0:1788575768.386711:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575768.386717:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575768.386720:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575768.386724:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386727:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386729:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386733:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575768.386736:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575768.386738:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575768.386740:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575768.386743:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575768.386747:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575768.386750:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575768.386755:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386757:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575768.386764:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386766:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575768.386769:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575768.386773:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575768.386777:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575768.386779:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575768.386782:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575768.386785:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386786:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575768.386789:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575768.386792:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386794:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575768.386799:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575768.386801:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575768.386804:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575768.386808:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575768.386810:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575768.386811:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575768.386814:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575768.386817:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575768.386820:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.386828:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810468352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575768.386836:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575768.386838:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.386840:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810468352/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575768.386847:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.386853:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16245) 00000100:00000040:0.0:1788575772.225745:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310911198, at 4310911232 00000100:00000010:0.0:1788575772.225757:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 10000000:00000001:0.0:1788575774.913854:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575774.913874:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575774.913877:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575774.913879:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575774.913881:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575774.913882:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575774.913884:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575774.913886:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575774.913888:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575774.913891:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575774.913893:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575774.913899:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575774.913904:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575774.913908:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575774.913911:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575774.913916:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575774.913923:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b000. 00000100:00000001:0.0:1788575774.913930:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575774.913934:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575774.913937:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575774.913943:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692b00. 00010000:00000001:0.0:1788575774.913947:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575774.913950:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30200. 00000020:00000001:0.0:1788575774.913954:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575774.913957:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30200 with handle 0x1441586093168dc9 to hash 00000020:00000001:0.0:1788575774.913959:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575774.913960:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788575774.913963:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788575774.913966:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575774.913969:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575774.913973:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575774.913983:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575774.914004:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575774.914011:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575774.914013:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575774.914019:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575774.914021:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575774.914023:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575774.914025:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788575774.914029:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575774.914034:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575774.914036:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575774.914038:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575774.914047:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575774.914055:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575774.914060:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575774.914066:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575774.914070:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575774.914077:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575774.914079:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575774.914082:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575774.914084:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575774.914086:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575774.914088:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575774.914092:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575774.914094:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575774.914096:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575774.914105:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575774.914111:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810469120:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575774.914114:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575774.914116:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575774.914117:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575774.914119:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575774.914120:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575774.914125:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575774.914126:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575774.914139:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575774.914143:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575774.914146:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575774.914149:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575774.914151:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575774.914153:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=4 00000020:00000001:0.0:1788575774.914155:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788575774.914158:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00010000:0.0:1788575774.914161:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914168:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575774.914171:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914177:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575774.914180:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914194:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575774.914196:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575774.914198:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914203:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575774.914205:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575774.914207:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575774.914209:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575774.914215:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575774.914217:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575774.914220:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914225:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575774.914226:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575774.914227:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575774.914228:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=5 00000020:00000001:0.0:1788575774.914230:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788575774.914232:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788575774.914234:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575774.914237:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575774.914242:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575774.914244:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575774.914248:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575774.914254:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575774.914256:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575774.914258:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575774.914260:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575774.914262:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575774.914264:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575774.914265:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575774.914267:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575774.914269:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30200 with handle 0x1441586093168dc9 from hash 00010000:00000001:0.0:1788575774.914272:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575774.914273:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575774.914274:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575774.914275:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575774.914276:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575774.914278:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575774.914280:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575774.914281:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575774.914283:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575774.914284:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575774.914287:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575774.914291:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575774.914292:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575774.914294:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575774.914295:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575774.914296:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575774.914297:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575774.914298:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575774.914300:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168dc9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575774.914306:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692b00 count: 0 00010000:00000001:0.0:1788575774.914313:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575774.914315:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575774.914317:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575774.914328:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810469120/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575774.914334:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575774.914336:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575774.914337:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575774.914339:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575774.914341:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b000. 02000000:00000001:0.0:1788575774.914345:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575774.914347:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575774.914349:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788575774.914352:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575774.914353:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575774.914355:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575774.914357:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575774.914359:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575778.753705:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310917727, at 4310917760 00000100:00000010:0.0:1788575778.753720:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00000040:0.0:1788575784.001759:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310922975, at 4310923008 00000100:00000010:0.0:1788575784.001783:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788575789.121722:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310928094, at 4310928128 00000100:00000010:0.0:1788575789.121736:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000800:00000200:0.0:1788575793.409962:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575794.241728:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310933215, at 4310933248 00000100:00000010:0.0:1788575794.241743:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 10000000:00000001:0.0:1788575795.009734:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575795.009747:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a99c0. 00000020:00000010:0.0:1788575795.009754:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788575795.009759:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575795.009763:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575795.009768:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575795.009773:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575795.009777:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575795.009784:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575795.009799:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985dc00. 10000000:00000040:0.0:1788575795.009804:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575795.009807:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575795.009812:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575795.009819:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575795.009822:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575795.009826:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575795.009829:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575795.009832:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788575795.009834:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985dc00. 00000020:00000010:0.0:1788575795.009837:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788575795.009840:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a99c0. 10000000:01000000:0.0:1788575795.009843:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575795.009846:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575795.009848:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575795.009850:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575795.009852:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575795.009855:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575795.009857:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575795.009859:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575795.009861:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575795.009863:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575795.009866:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575795.009868:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575795.009870:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575796.801739:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575796.801745:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575796.801748:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575796.801750:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575796.801752:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575796.801752:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575796.801755:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575796.801756:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575796.801758:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575796.801760:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575796.801761:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575796.801781:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575796.801785:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575796.801789:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575796.801791:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575796.801795:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575796.801801:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788575796.801805:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575796.801809:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575796.801811:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575796.801816:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692b00. 00010000:00000001:0.0:1788575796.801819:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575796.801822:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788575796.801825:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575796.801834:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168dd0 to hash 00000020:00000001:0.0:1788575796.801836:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575796.801837:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788575796.801839:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788575796.801841:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575796.801843:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575796.801846:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575796.801854:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575796.801862:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575796.801865:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575796.801866:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575796.801871:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575796.801872:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575796.801874:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575796.801875:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788575796.801882:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575796.801885:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575796.801887:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575796.801888:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575796.801896:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575796.801903:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575796.801905:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575796.801909:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575796.801912:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575796.801915:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575796.801917:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575796.801918:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575796.801919:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575796.801921:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575796.801923:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575796.801925:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575796.801926:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575796.801928:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575796.801930:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575796.801935:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810472448:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575796.801938:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575796.801940:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575796.801941:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575796.801943:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575796.801945:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575796.801961:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575796.801964:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575796.801969:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575796.801970:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575796.801974:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575796.801977:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575796.801978:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575796.801980:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788575796.801983:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788575796.801985:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788575796.801989:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.801995:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575796.801999:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.802002:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575796.802004:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.802008:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575796.802009:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575796.802010:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.802014:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575796.802015:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575796.802017:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575796.802018:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575796.802029:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575796.802030:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575796.802032:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.802035:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575796.802036:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575796.802037:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575796.802038:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788575796.802039:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788575796.802040:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788575796.802042:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575796.802043:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575796.802047:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575796.802060:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575796.802064:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575796.802065:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575796.802066:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575796.802068:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575796.802069:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575796.802071:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575796.802072:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575796.802073:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575796.802075:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575796.802076:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168dd0 from hash 00010000:00000001:0.0:1788575796.802078:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575796.802079:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575796.802079:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575796.802083:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575796.802084:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575796.802085:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575796.802087:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575796.802087:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575796.802088:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575796.802089:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575796.802091:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575796.802094:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575796.802095:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575796.802096:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575796.802097:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575796.802098:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575796.802099:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575796.802099:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575796.802101:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168dd0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575796.802104:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692b00 count: 0 00010000:00000001:0.0:1788575796.802111:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575796.802112:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575796.802114:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575796.802116:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810472448/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575796.802120:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575796.802122:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575796.802123:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575796.802125:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575796.802127:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788575796.802130:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575796.802132:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575796.802134:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788575796.802137:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575796.802138:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575796.802144:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575796.802146:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575796.802149:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575800.641760:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310939615, at 4310939648 00000100:00000010:0.0:1788575800.641774:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00000040:0.0:1788575806.017765:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310944990, at 4310945024 00000100:00000010:0.0:1788575806.017783:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00080000:0.0:1788575807.297791:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575807.297806:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575807.297809:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575807.297812:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575807.297817:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575807.297824:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16273 00000400:00000200:0.0:1788575807.297833:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575807.297839:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575807.297842:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575807.297845:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.297847:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.297850:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.297852:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575807.297855:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575807.297856:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575807.297859:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575807.297862:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575807.297863:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575807.297865:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575807.297867:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.297869:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.297870:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.297871:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575807.297891:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575807.297894:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575807.297896:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575807.297899:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575807.297901:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575807.297903:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.297905:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575807.297907:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575807.297912:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575807.297916:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575807.297922:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575807.297925:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.297930:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575807.297934:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575807.297940:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575807.297951:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575807.297962:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.297975:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810473856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575807.297996:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575807.298000:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298005:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810473856/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575807.298023:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575807.298025:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575807.298030:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575807.298033:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575807.298035:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575807.298039:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575807.298043:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16273 00000400:00000200:0.0:1788575807.298050:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575807.298072:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575807.298077:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575807.298082:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298086:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298090:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298095:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575807.298100:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575807.298103:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575807.298107:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575807.298112:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575807.298119:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575807.298124:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575807.298132:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298138:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298144:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298150:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575807.298158:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575807.298164:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298170:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575807.298174:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575807.298183:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575807.298188:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298190:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575807.298196:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575807.298199:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298201:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575807.298206:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575807.298210:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575807.298214:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575807.298216:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298218:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575807.298224:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575807.298227:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575807.298231:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575807.298234:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.298241:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810473984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575807.298249:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575807.298253:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298259:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810473984/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575807.298277:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575807.298281:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575807.298286:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575807.298290:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575807.298291:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575807.298293:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575807.298298:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16273 00000400:00000200:0.0:1788575807.298303:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575807.298307:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575807.298309:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575807.298315:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298317:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298319:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298321:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575807.298322:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575807.298324:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575807.298326:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575807.298330:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575807.298332:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575807.298334:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575807.298343:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298345:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298348:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298352:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575807.298353:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575807.298359:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298362:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575807.298366:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575807.298368:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575807.298371:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298373:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575807.298377:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575807.298380:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298383:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575807.298387:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575807.298391:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575807.298394:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575807.298396:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298399:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575807.298400:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575807.298402:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575807.298407:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575807.298412:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.298420:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810474112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575807.298428:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575807.298431:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298437:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810474112/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575807.298452:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575807.298456:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575807.298481:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575807.298484:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575807.298485:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575807.298489:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575807.298495:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16273 00000400:00000200:0.0:1788575807.298502:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575807.298506:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575807.298509:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575807.298514:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298519:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298524:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298530:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575807.298533:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575807.298535:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575807.298538:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575807.298541:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575807.298545:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575807.298549:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575807.298553:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298557:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298561:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298563:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575807.298567:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575807.298571:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298576:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575807.298579:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575807.298582:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575807.298585:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298590:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575807.298594:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575807.298604:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298609:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575807.298615:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575807.298620:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575807.298623:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575807.298626:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298628:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575807.298630:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575807.298632:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575807.298682:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575807.298688:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.298694:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810474240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575807.298704:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575807.298708:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298714:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810474240/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575807.298731:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575807.298735:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575807.298738:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575807.298740:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575807.298744:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575807.298746:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575807.298749:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16273 00000400:00000200:0.0:1788575807.298754:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575807.298769:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575807.298775:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575807.298781:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298785:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298789:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298800:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575807.298806:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575807.298809:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575807.298812:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575807.298815:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575807.298819:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575807.298822:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575807.298827:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298830:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575807.298834:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298836:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575807.298838:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575807.298842:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575807.298845:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575807.298848:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575807.298851:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575807.298855:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298858:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575807.298860:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575807.298863:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298864:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575807.298869:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575807.298872:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575807.298876:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575807.298878:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.298881:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575807.298883:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575807.298886:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575807.298891:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575807.298895:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.298902:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810474368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575807.298923:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575807.298928:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298934:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810474368/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575807.298942:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575807.298946:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16284) 00000100:00000040:0.0:1788575811.137842:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310950110, at 4310950144 00000100:00000010:0.0:1788575811.137849:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788575812.417720:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575812.417731:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575812.417733:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575812.417737:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575812.417742:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575812.417749:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16279 00000400:00000200:0.0:1788575812.417757:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575812.417762:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575812.417765:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575812.417769:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417772:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.417774:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417776:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575812.417780:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575812.417782:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575812.417785:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575812.417788:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575812.417790:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575812.417792:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575812.417795:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417812:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.417814:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417816:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575812.417820:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575812.417822:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417825:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575812.417828:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575812.417831:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575812.417834:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.417835:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575812.417838:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575812.417843:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575812.417847:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575812.417850:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575812.417851:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.417854:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575812.417857:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575812.417861:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575812.417868:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575812.417876:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.417885:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810474496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575812.417895:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575812.417899:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.417903:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810474496/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575812.417926:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575812.417929:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575812.417932:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575812.417934:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575812.417935:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575812.417937:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575812.417946:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16279 00000400:00000200:0.0:1788575812.417950:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575812.417953:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575812.417955:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575812.417970:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417973:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.417976:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.417979:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575812.417983:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575812.417986:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575812.417989:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575812.417992:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575812.417995:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575812.417998:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575812.418006:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418010:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418015:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418019:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575812.418025:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575812.418029:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418034:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575812.418039:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575812.418047:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575812.418052:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418056:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575812.418061:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575812.418065:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418069:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575812.418075:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575812.418083:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575812.418088:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575812.418098:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418101:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575812.418103:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575812.418107:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575812.418113:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575812.418117:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.418125:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810474624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575812.418134:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575812.418136:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.418139:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810474624/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575812.418149:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575812.418153:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575812.418158:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575812.418161:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575812.418163:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575812.418167:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575812.418170:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16279 00000400:00000200:0.0:1788575812.418174:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575812.418176:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575812.418179:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575812.418181:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418183:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418185:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418187:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575812.418189:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575812.418190:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575812.418191:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575812.418193:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575812.418195:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575812.418201:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575812.418203:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418205:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418207:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418209:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575812.418211:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575812.418213:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418216:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575812.418217:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575812.418220:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575812.418222:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418223:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575812.418225:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575812.418228:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418229:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575812.418232:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575812.418234:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575812.418236:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575812.418237:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418239:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575812.418240:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575812.418241:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575812.418244:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575812.418247:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.418251:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810474752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575812.418257:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575812.418259:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.418263:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810474752/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575812.418270:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575812.418287:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575812.418291:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575812.418292:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575812.418294:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575812.418296:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575812.418298:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16279 00000400:00000200:0.0:1788575812.418302:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575812.418305:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575812.418307:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575812.418309:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418311:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418313:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418318:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575812.418322:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575812.418325:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575812.418327:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575812.418332:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575812.418335:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575812.418339:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575812.418344:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418348:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418351:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418356:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575812.418358:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575812.418362:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418365:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575812.418370:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575812.418372:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575812.418374:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418376:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575812.418384:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575812.418389:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418394:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575812.418397:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575812.418402:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575812.418407:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575812.418410:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418412:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575812.418414:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575812.418418:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575812.418423:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575812.418429:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.418437:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810474880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575812.418448:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575812.418450:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.418459:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810474880/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575812.418474:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575812.418478:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575812.418482:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575812.418484:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575812.418488:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575812.418490:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575812.418495:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16279 00000400:00000200:0.0:1788575812.418500:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575812.418506:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575812.418511:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575812.418516:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418520:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418524:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418535:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575812.418540:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575812.418543:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575812.418546:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575812.418549:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575812.418554:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575812.418558:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575812.418564:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418567:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575812.418569:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418574:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575812.418580:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575812.418583:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575812.418588:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575812.418591:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575812.418594:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575812.418598:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418603:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575812.418605:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575812.418609:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418611:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575812.418628:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575812.418631:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575812.418633:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575812.418652:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575812.418654:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575812.418655:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575812.418657:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575812.418660:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575812.418663:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.418668:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810475008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575812.418683:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575812.418685:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.418688:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810475008/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575812.418695:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575812.418699:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16289) 00000100:00000040:0.0:1788575816.257748:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310955231, at 4310955264 00000100:00000010:0.0:1788575816.257766:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00080000:0.0:1788575817.537823:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575817.537832:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575817.537835:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575817.537839:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575817.537844:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575817.537852:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16284 00000400:00000200:0.0:1788575817.537862:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575817.537868:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575817.537872:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575817.537876:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.537880:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.537883:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.537886:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575817.537890:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575817.537893:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575817.537896:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575817.537899:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575817.537902:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575817.537904:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575817.537908:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.537918:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.537922:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.537924:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575817.537927:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575817.537932:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575817.537935:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575817.537938:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575817.537941:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575817.537944:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.537947:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575817.537950:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575817.537956:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575817.537960:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575817.537964:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575817.537966:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.537969:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575817.537971:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575817.537975:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575817.537982:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575817.537990:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.538000:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810475136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575817.538011:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575817.538015:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.538021:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810475136/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575817.538043:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575817.538046:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575817.538050:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575817.538052:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575817.538053:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575817.538056:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575817.538083:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16284 00000400:00000200:0.0:1788575817.538091:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575817.538097:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575817.538102:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575817.538107:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538112:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.538115:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538120:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575817.538132:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575817.538135:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575817.538139:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575817.538143:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575817.538145:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575817.538149:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575817.538156:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538158:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.538161:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538166:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575817.538172:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575817.538181:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538185:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575817.538189:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575817.538196:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575817.538200:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.538202:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575817.538208:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575817.538213:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.538216:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575817.538222:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575817.538227:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575817.538233:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575817.538240:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.538246:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575817.538248:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575817.538252:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575817.538259:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575817.538265:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.538274:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810475264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575817.538286:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575817.538290:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.538296:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810475264/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575817.538309:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575817.538314:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575817.538319:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575817.538322:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575817.538325:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575817.538329:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575817.538334:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16284 00000400:00000200:0.0:1788575817.538356:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575817.538361:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575817.538366:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575817.538370:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538375:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.538381:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538384:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575817.538388:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575817.538390:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575817.538399:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575817.538404:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575817.538408:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575817.538418:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575817.538426:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538429:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.538433:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538436:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575817.538440:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575817.538443:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575817.538446:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575817.538450:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575817.538453:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575817.538458:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.538463:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575817.538968:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575817.538975:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.538979:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575817.538986:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575817.538991:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575817.538997:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575817.539000:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539004:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575817.539008:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575817.539011:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575817.539018:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575817.539025:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.539037:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810475392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575817.539054:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575817.539069:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.539081:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810475392/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575817.539100:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575817.539111:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575817.539117:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575817.539121:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575817.539124:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575817.539131:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575817.539137:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16284 00000400:00000200:0.0:1788575817.539146:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575817.539155:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575817.539160:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575817.539167:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539172:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.539177:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539182:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575817.539186:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575817.539189:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575817.539193:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575817.539196:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575817.539201:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575817.539206:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575817.539213:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539219:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.539224:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539228:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575817.539233:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575817.539239:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539244:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575817.539248:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575817.539253:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575817.539257:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539262:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575817.539271:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575817.539278:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539282:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575817.539288:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575817.539294:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575817.539300:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575817.539304:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539308:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575817.539311:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575817.539315:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575817.539323:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575817.539330:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.539340:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810475520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575817.539356:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575817.539360:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.539368:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810475520/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575817.539393:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575817.539399:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575817.539408:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575817.539413:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575817.539416:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575817.539422:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575817.539427:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16284 00000400:00000200:0.0:1788575817.539437:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575817.539446:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575817.539452:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575817.539459:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539484:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.539490:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539499:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575817.539509:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575817.539512:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575817.539517:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575817.539521:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575817.539526:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575817.539532:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575817.539541:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539545:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575817.539550:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539554:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575817.539558:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575817.539566:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575817.539572:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575817.539576:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575817.539581:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575817.539586:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539591:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575817.539593:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575817.539598:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539601:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575817.539607:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575817.539611:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575817.539616:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575817.539619:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575817.539622:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575817.539626:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575817.539629:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575817.539654:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575817.539661:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.539670:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810475648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575817.539699:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575817.539704:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.539711:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810475648/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575817.539727:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575817.539733:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16294) 00000100:00080000:0.0:1788575818.114176:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575818.114182:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575818.114191:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575818.114194:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575818.114197:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575818.114201:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575818.114208:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575818.114213:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575818.114216:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575818.114219:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575818.114222:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575818.114225:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575818.114227:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575818.114230:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575818.114232:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575818.114233:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575818.114235:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575818.114238:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575818.114240:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575818.114243:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575818.114245:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575818.114247:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575818.114249:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575818.114259:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575818.114268:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575818.114272:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575818.114274:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575818.114276:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575818.114279:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575818.114281:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575818.114283:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575818.114288:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575818.114290:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575818.114293:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575818.114295:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575818.114297:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575818.114299:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575818.114301:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575818.114305:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575818.114308:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575818.114314:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810475904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575818.114323:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575818.114326:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575818.114330:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810475904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575818.114350:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575818.114356:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16294) 00000100:00000040:0.0:1788575821.953741:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310960926, at 4310960960 00000100:00000010:0.0:1788575821.953755:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00080000:0.0:1788575823.169768:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575823.169780:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575823.169782:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575823.169786:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575823.169790:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575823.169812:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575823.169820:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575823.169825:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575823.169828:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575823.169831:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169834:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.169837:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169839:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575823.169843:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575823.169845:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575823.169848:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575823.169850:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575823.169852:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575823.169853:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575823.169856:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169857:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.169858:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169859:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575823.169861:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575823.169864:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169865:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575823.169867:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575823.169870:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575823.169872:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.169873:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575823.169874:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575823.169879:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575823.169882:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575823.169885:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575823.169886:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.169888:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575823.169890:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575823.169897:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575823.169903:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575823.169908:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.169914:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810476032/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575823.169922:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575823.169925:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.169928:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810476032/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575823.169945:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575823.169947:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575823.169949:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575823.169950:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575823.169951:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575823.169953:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575823.169954:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575823.169957:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575823.169959:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575823.169960:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575823.169962:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169963:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.169964:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169965:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575823.169967:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575823.169968:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575823.169969:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575823.169970:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575823.169972:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575823.169973:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575823.169975:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169980:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.169982:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169983:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575823.169985:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575823.169987:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575823.169990:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575823.169993:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575823.169998:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575823.170002:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170004:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575823.170007:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575823.170010:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170014:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575823.170017:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575823.170022:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575823.170026:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575823.170028:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170032:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575823.170035:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575823.170038:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575823.170044:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575823.170049:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.170058:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810476160/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575823.170066:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575823.170070:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.170076:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810476160/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575823.170091:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575823.170094:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575823.170100:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575823.170108:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575823.170112:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575823.170116:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575823.170122:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575823.170130:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575823.170136:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575823.170138:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575823.170142:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170146:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170149:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170153:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575823.170158:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575823.170161:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575823.170165:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575823.170169:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575823.170174:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575823.170179:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575823.170185:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170189:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170195:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170199:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575823.170202:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575823.170207:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170213:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575823.170214:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575823.170218:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575823.170220:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170222:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575823.170226:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575823.170230:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170237:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575823.170242:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575823.170247:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575823.170253:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575823.170256:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170260:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575823.170264:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575823.170267:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575823.170274:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575823.170280:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.170287:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810476288/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575823.170300:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575823.170303:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.170309:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810476288/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575823.170332:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575823.170335:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575823.170339:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575823.170343:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575823.170345:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575823.170347:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575823.170352:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575823.170359:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575823.170364:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575823.170366:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575823.170370:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170371:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170372:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170377:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575823.170380:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575823.170386:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575823.170390:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575823.170395:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575823.170400:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575823.170405:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575823.170414:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170419:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170426:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170432:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575823.170437:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575823.170445:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170451:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575823.170456:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575823.170462:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575823.170467:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170470:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575823.170474:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575823.170478:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170482:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575823.170490:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575823.170496:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575823.170500:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575823.170503:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170507:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575823.170510:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575823.170513:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575823.170520:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575823.170526:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.170535:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810476416/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575823.170549:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575823.170554:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.170572:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810476416/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575823.170597:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575823.170602:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575823.170609:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575823.170613:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575823.170616:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575823.170623:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575823.170629:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16289 00000400:00000200:0.0:1788575823.170667:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575823.170675:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575823.170681:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575823.170690:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170694:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170696:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170698:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575823.170704:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575823.170709:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575823.170714:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575823.170718:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575823.170723:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575823.170729:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575823.170736:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170742:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575823.170749:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170753:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575823.170758:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575823.170764:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575823.170769:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575823.170778:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575823.170784:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575823.170790:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170794:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575823.170799:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575823.170804:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170811:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575823.170816:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575823.170819:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575823.170826:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575823.170829:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.170834:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575823.170837:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575823.170841:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575823.170847:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575823.170854:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.170865:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810476544/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575823.170878:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575823.170884:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.170894:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810476544/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575823.170915:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575823.170923:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16299) 00000100:00000040:0.0:1788575827.009756:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310965982, at 4310966016 00000100:00000010:0.0:1788575827.009773:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00080000:0.0:1788575828.289794:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575828.289804:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575828.289807:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575828.289810:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575828.289816:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575828.289823:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16294 00000400:00000200:0.0:1788575828.289848:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575828.289855:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575828.289859:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575828.289864:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.289867:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.289870:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.289872:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575828.289877:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575828.289879:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575828.289882:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575828.289885:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575828.289888:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575828.289890:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575828.289893:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.289895:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.289897:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.289899:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575828.289902:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575828.289906:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575828.289908:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575828.289911:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575828.289914:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575828.289917:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.289920:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575828.289922:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575828.289930:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575828.289934:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575828.289937:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575828.289939:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.289942:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575828.289945:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575828.289948:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575828.289960:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575828.289966:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575828.289976:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810476672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575828.289986:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575828.289990:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.289994:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810476672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575828.290016:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575828.290020:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575828.290023:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575828.290024:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575828.290026:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575828.290029:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575828.290031:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16294 00000400:00000200:0.0:1788575828.290035:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575828.290039:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575828.290042:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575828.290046:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290048:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290050:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290053:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575828.290056:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575828.290057:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575828.290062:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575828.290064:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575828.290069:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575828.290073:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575828.290079:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290089:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290094:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290098:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575828.290102:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575828.290107:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290111:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575828.290117:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575828.290123:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575828.290128:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290132:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575828.290138:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575828.290144:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290148:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575828.290154:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575828.290157:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575828.290163:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575828.290165:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290167:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575828.290170:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575828.290174:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575828.290194:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575828.290198:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575828.290204:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810476800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575828.290215:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575828.290218:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.290221:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810476800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575828.290232:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575828.290236:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575828.290241:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575828.290250:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575828.290252:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575828.290257:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575828.290263:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16294 00000400:00000200:0.0:1788575828.290271:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575828.290277:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575828.290282:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575828.290289:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290294:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290297:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290301:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575828.290305:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575828.290308:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575828.290311:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575828.290313:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575828.290316:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575828.290321:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575828.290326:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290329:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290334:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290338:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575828.290342:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575828.290349:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290353:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575828.290358:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575828.290364:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575828.290369:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290373:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575828.290377:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575828.290382:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290392:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575828.290398:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575828.290403:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575828.290408:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575828.290410:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290413:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575828.290415:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575828.290417:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575828.290424:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575828.290429:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575828.290437:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810476928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575828.290452:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575828.290456:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.290464:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810476928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575828.290475:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575828.290482:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575828.290489:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575828.290490:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575828.290494:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575828.290498:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575828.290503:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16294 00000400:00000200:0.0:1788575828.290520:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575828.290533:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575828.290538:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575828.290545:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290548:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290553:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290557:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575828.290560:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575828.290570:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575828.290573:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575828.290576:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575828.290581:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575828.290584:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575828.290589:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290594:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290596:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290602:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575828.290606:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575828.290612:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290616:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575828.290621:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575828.290626:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575828.290629:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290631:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575828.290633:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575828.290666:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290670:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575828.290676:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575828.290681:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575828.290685:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575828.290688:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290692:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575828.290695:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575828.290699:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575828.290706:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575828.290713:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575828.290722:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810477056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575828.290738:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575828.290743:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.290757:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810477056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575828.290778:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575828.290782:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575828.290789:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575828.290793:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575828.290797:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575828.290802:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575828.290807:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16294 00000400:00000200:0.0:1788575828.290816:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575828.290823:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575828.290827:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575828.290833:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290837:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290839:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290841:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575828.290845:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575828.290848:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575828.290851:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575828.290855:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575828.290857:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575828.290860:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575828.290864:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290869:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575828.290874:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290878:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575828.290884:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575828.290891:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575828.290897:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575828.290905:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575828.290909:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575828.290914:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290919:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575828.290923:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575828.290929:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290933:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575828.290939:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575828.290946:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575828.290951:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575828.290955:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575828.290959:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575828.290963:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575828.290965:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575828.290972:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575828.290978:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575828.290986:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810477184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575828.290999:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575828.291003:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.291010:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810477184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575828.291026:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575828.291034:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16304) 00000800:00000200:0.0:1788575831.297953:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575832.129753:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310971102, at 4310971136 00000100:00000010:0.0:1788575832.129768:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000100:00000040:0.0:1788575837.249754:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310976222, at 4310976256 00000100:00000010:0.0:1788575837.249775:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 10000000:00000001:0.0:1788575840.449783:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575840.449789:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575840.449800:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575840.449803:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575840.449820:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575840.449821:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575840.449823:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575840.449825:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575840.449827:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575840.449831:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575840.449833:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575840.449855:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575840.449861:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575840.449868:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575840.449873:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575840.449879:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575840.449888:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788575840.449894:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575840.449898:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575840.449901:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575840.449907:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692400. 00010000:00000001:0.0:1788575840.449912:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575840.449915:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30800. 00000020:00000001:0.0:1788575840.449919:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575840.449921:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30800 with handle 0x1441586093168dde to hash 00000020:00000001:0.0:1788575840.449923:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575840.449924:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575840.449927:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575840.449931:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575840.449934:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575840.449938:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575840.449948:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575840.449958:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575840.449974:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575840.449976:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575840.449982:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575840.449984:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575840.449987:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575840.449988:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531982336 : -109714177569280 : ffff9c372d6b4e00) 00000100:00000001:0.0:1788575840.449992:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575840.449997:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575840.449999:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575840.450000:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575840.450010:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575840.450019:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575840.450023:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575840.450030:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575840.450034:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575840.450041:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575840.450043:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575840.450045:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575840.450047:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575840.450050:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575840.450052:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575840.450056:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575840.450058:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575840.450060:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575840.450065:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575840.450072:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810478592:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575840.450080:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575840.450082:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575840.450083:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575840.450085:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575840.450087:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575840.450092:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575840.450093:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575840.450097:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575840.450098:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575840.450100:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575840.450102:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575840.450104:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575840.450106:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=4 00000020:00000001:0.0:1788575840.450107:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575840.450109:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00010000:0.0:1788575840.450111:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450115:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575840.450117:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450121:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575840.450123:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450127:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575840.450128:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575840.450129:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450138:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575840.450139:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575840.450141:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575840.450143:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575840.450146:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30800 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575840.450148:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575840.450149:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450152:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575840.450153:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575840.450154:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575840.450155:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30800 refcount=5 00000020:00000001:0.0:1788575840.450156:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575840.450157:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536683008 : -109714172868608 : ffff9c372db30800) 00010000:00000001:0.0:1788575840.450159:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575840.450161:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575840.450164:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575840.450165:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575840.450168:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575840.450170:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575840.450171:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575840.450172:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575840.450173:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575840.450175:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575840.450180:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575840.450181:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575840.450183:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575840.450185:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30800 with handle 0x1441586093168dde from hash 00010000:00000001:0.0:1788575840.450187:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575840.450188:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575840.450190:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575840.450191:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575840.450192:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575840.450194:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575840.450196:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575840.450197:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575840.450198:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575840.450200:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575840.450203:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575840.450209:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575840.450210:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575840.450211:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575840.450212:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575840.450214:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575840.450215:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575840.450216:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575840.450218:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30800/0x1441586093168dde lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575840.450225:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692400 count: 0 00010000:00000001:0.0:1788575840.450234:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575840.450236:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575840.450238:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575840.450240:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4e00 x1875440810478592/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575840.450245:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575840.450247:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575840.450248:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575840.450253:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575840.450255:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788575840.450258:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575840.450259:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575840.450261:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4e00. 00000100:00000001:0.0:1788575840.450264:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575840.450264:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575840.450266:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575840.450267:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575840.450269:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575844.289766:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310983263, at 4310983296 00000100:00000010:0.0:1788575844.289782:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00080000:0.0:1788575845.697775:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575845.697785:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575845.697787:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575845.697791:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575845.697795:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575845.697802:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16312 00000400:00000200:0.0:1788575845.697811:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575845.697816:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575845.697820:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575845.697822:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.697826:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.697828:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.697830:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575845.697835:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575845.697837:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575845.697839:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575845.697842:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575845.697844:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575845.697846:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575845.697849:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.697867:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.697869:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.697871:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575845.697874:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575845.697877:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575845.697880:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575845.697882:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575845.697886:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575845.697888:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.697891:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575845.697893:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575845.697898:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575845.697901:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575845.697905:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575845.697906:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.697909:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575845.697911:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575845.697915:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575845.697921:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575845.697927:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575845.697940:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810479360/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575845.697949:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575845.697953:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.697957:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810479360/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575845.697975:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575845.697978:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575845.697981:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575845.697982:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575845.697984:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575845.697986:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575845.697993:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16312 00000400:00000200:0.0:1788575845.697997:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575845.698001:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575845.698004:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575845.698007:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698009:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698012:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698016:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575845.698021:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575845.698024:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575845.698026:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575845.698030:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575845.698032:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575845.698038:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575845.698046:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698048:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698050:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698052:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575845.698054:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575845.698059:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698064:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575845.698068:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575845.698076:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575845.698081:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698085:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575845.698091:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575845.698095:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698099:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575845.698104:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575845.698109:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575845.698114:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575845.698121:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698125:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575845.698128:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575845.698130:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575845.698136:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575845.698141:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575845.698150:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810479488/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575845.698164:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575845.698168:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.698177:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810479488/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575845.698188:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575845.698190:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575845.698195:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575845.698196:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575845.698200:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575845.698204:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575845.698212:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16312 00000400:00000200:0.0:1788575845.698219:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575845.698222:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575845.698224:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575845.698231:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698236:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698242:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698245:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575845.698248:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575845.698251:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575845.698254:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575845.698258:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575845.698262:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575845.698272:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575845.698280:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698284:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698286:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698290:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575845.698296:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575845.698302:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698308:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575845.698313:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575845.698317:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575845.698321:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698326:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575845.698329:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575845.698334:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698337:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575845.698343:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575845.698348:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575845.698354:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575845.698357:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698361:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575845.698363:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575845.698366:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575845.698372:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575845.698375:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575845.698383:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810479616/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575845.698397:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575845.698401:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.698408:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810479616/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575845.698421:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575845.698430:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575845.698433:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575845.698435:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575845.698436:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575845.698439:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575845.698441:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16312 00000400:00000200:0.0:1788575845.698445:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575845.698448:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575845.698450:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575845.698452:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698454:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698456:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698458:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575845.698460:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575845.698462:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575845.698463:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575845.698465:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575845.698466:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575845.698468:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575845.698471:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698472:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698474:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698476:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575845.698478:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575845.698480:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698482:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575845.698484:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575845.698487:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575845.698489:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698491:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575845.698506:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575845.698508:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698513:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575845.698516:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575845.698518:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575845.698521:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575845.698522:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698524:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575845.698525:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575845.698528:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575845.698545:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788575845.698554:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575845.698564:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810479744/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575845.698576:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575845.698580:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.698585:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810479744/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575845.698595:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575845.698601:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575845.698607:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575845.698611:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575845.698616:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575845.698620:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575845.698625:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16312 00000400:00000200:0.0:1788575845.698632:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575845.698704:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575845.698711:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575845.698717:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698723:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698728:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698737:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575845.698741:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575845.698744:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575845.698747:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575845.698751:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575845.698755:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575845.698759:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575845.698765:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698771:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575845.698775:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698778:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575845.698783:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575845.698788:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575845.698794:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575845.698798:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575845.698803:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575845.698808:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698812:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575845.698814:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575845.698817:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698818:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575845.698822:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575845.698827:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575845.698831:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575845.698834:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575845.698839:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575845.698841:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575845.698846:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575845.698853:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788575845.698859:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575845.698868:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810479872/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575845.698902:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575845.698907:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.698915:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810479872/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575845.698929:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.698937:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16322) 00000100:00000040:0.0:1788575849.539972:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310988510, at 4310988546 00000100:00000010:0.0:1788575849.539984:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00000040:0.0:1788575854.657764:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310993630, at 4310993664 00000100:00000010:0.0:1788575854.657781:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000100:00000040:0.0:1788575859.777896:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4310998750, at 4310998784 00000100:00000010:0.0:1788575859.777912:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb720. 00000100:00000040:0.0:1788575866.049751:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311005023, at 4311005056 00000100:00000010:0.0:1788575866.049758:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00000040:0.0:1788575871.553747:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311010527, at 4311010560 00000100:00000010:0.0:1788575871.553763:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215de0. 00000100:00000040:0.0:1788575876.673764:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311015646, at 4311015680 00000100:00000010:0.0:1788575876.673780:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215120. 00000100:00000040:0.0:1788575881.793742:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311020766, at 4311020800 00000100:00000010:0.0:1788575881.793765:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215840. 10000000:00000001:0.0:1788575882.561732:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575882.561744:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9740. 00000020:00000010:0.0:1788575882.561750:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985dc00. 00000020:00000010:0.0:1788575882.561754:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985d600. 00000040:00000010:0.0:1788575882.561756:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575882.561760:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575882.561766:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788575882.561769:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575882.561782:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575882.561785:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788575882.561789:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575882.561791:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575882.561794:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985d600. 00000040:00000010:0.0:1788575882.561801:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985cc00. 00000020:00000010:0.0:1788575882.561804:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575882.561806:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4be00. 80000000:00000010:0.0:1788575882.561809:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626120. 00040000:00000010:0.0:1788575882.561811:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ae00. 20000000:00000010:0.0:1788575882.561813:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575882.561815:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985dc00. 10000000:00000010:0.0:1788575882.561816:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9740. 10000000:01000000:0.0:1788575882.561818:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575882.561820:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575882.561822:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575882.561823:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575882.561824:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575882.561826:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575882.561827:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575882.561829:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575882.561830:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575882.561832:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575882.561834:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575882.561835:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575882.561837:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00080000:0.0:1788575883.073845:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575883.073853:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575883.073855:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575883.073858:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575883.073861:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575883.073874:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16349 00000400:00000200:0.0:1788575883.073882:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575883.073887:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575883.073905:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575883.073908:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.073910:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.073911:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.073913:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575883.073915:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575883.073917:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575883.073919:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.073922:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575883.073924:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575883.073926:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575883.073936:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.073938:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.073940:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.073942:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575883.073945:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575883.073948:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575883.073951:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575883.073954:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575883.073957:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575883.073960:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.073962:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575883.073964:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.073970:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575883.073974:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575883.073978:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575883.073980:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.073983:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.073985:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.073989:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.073997:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575883.074005:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.074019:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810484608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575883.074029:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.074033:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.074038:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810484608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.074621:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575883.074624:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575883.074626:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575883.074627:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575883.074628:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575883.074630:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575883.074633:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16349 00000400:00000200:0.0:1788575883.074667:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575883.074670:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575883.074671:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575883.074673:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.074674:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.074676:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.074677:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575883.074678:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575883.074679:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575883.074681:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.074682:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575883.074684:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575883.074686:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575883.074688:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.074689:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.074691:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.074697:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575883.074699:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575883.074701:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575883.074703:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575883.074704:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575883.074708:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575883.074710:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.074711:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575883.074714:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575883.074716:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.074718:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.074720:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575883.074722:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575883.074724:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575883.074725:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.074727:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.074728:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.074730:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.074732:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575883.074735:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.074738:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810484736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575883.074744:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.074746:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.074749:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810484736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.075322:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575883.075325:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575883.075327:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575883.075331:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575883.075332:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575883.075338:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575883.075340:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16349 00000400:00000200:0.0:1788575883.075343:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575883.075345:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575883.075347:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575883.075349:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075351:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075352:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075354:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575883.075355:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575883.075356:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575883.075357:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.075362:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575883.075365:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575883.075368:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575883.075370:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075371:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075373:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075374:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575883.075375:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575883.075377:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075379:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575883.075380:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575883.075382:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575883.075383:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075385:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575883.075386:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575883.075387:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075388:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.075390:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575883.075394:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575883.075396:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575883.075397:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075398:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.075399:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.075401:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.075403:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788575883.075406:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.075409:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810484864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575883.075414:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.075415:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.075417:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810484864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.075428:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575883.075430:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575883.075432:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575883.075433:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575883.075435:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575883.075436:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575883.075437:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16349 00000400:00000200:0.0:1788575883.075439:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575883.075441:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575883.075442:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575883.075444:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075445:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075446:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075447:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575883.075449:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575883.075450:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575883.075454:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.075455:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575883.075457:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575883.075458:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575883.075460:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075461:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075462:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075463:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575883.075464:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575883.075465:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075467:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575883.075467:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575883.075469:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575883.075470:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075471:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575883.075473:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575883.075474:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075475:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.075478:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575883.075480:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575883.075482:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575883.075483:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075484:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.075486:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.075490:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.075575:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788575883.075580:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.075589:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810484992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575883.075601:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.075603:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.075609:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810484992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.075629:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575883.075633:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575883.075651:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575883.075654:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575883.075655:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575883.075659:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575883.075664:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16349 00000400:00000200:0.0:1788575883.075669:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575883.075674:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575883.075677:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575883.075682:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075686:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075689:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075692:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575883.075693:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575883.075694:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575883.075695:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.075696:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575883.075697:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575883.075698:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575883.075700:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075705:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575883.075707:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075709:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575883.075712:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575883.075715:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575883.075719:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575883.075725:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575883.075729:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575883.075734:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075737:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575883.075739:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575883.075741:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075746:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.075750:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575883.075755:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575883.075760:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575883.075764:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.075770:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.075774:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.075778:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.075785:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788575883.075791:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.075800:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810485120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575883.075816:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.075820:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.075828:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810485120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.075851:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.075856:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16359) 10000000:00000001:0.0:1788575883.649756:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575883.649763:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575883.649768:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575883.649770:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575883.649772:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575883.649773:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575883.649776:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575883.649778:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575883.649781:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575883.649784:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575883.649800:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575883.649807:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575883.649811:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575883.649815:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575883.649818:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575883.649824:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575883.649831:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575883.649838:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575883.649842:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575883.649845:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575883.649852:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692700. 00010000:00000001:0.0:1788575883.649857:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575883.649860:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31600. 00000020:00000001:0.0:1788575883.649865:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575883.649868:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31600 with handle 0x1441586093168dec to hash 00000020:00000001:0.0:1788575883.649870:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575883.649872:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575883.649875:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575883.649879:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575883.649882:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575883.649886:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575883.649895:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575883.649906:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575883.649912:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575883.649947:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575883.649955:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575883.649957:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185612544 : -109714523939072 : ffff9c3718c61f00) 00000100:00000001:0.0:1788575883.649960:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575883.649966:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788575883.649970:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575883.649980:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575883.649982:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575883.649984:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575883.649997:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.650005:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575883.650009:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.650015:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575883.650018:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.650024:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575883.650027:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575883.650029:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.650031:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61f00 going to sleep for 0 seconds 00000100:00000001:0.0:1788575883.650034:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575883.650036:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575883.650039:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575883.650041:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575883.650043:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575883.650047:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575883.650053:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810485248:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575883.650058:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575883.650061:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575883.650063:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575883.650065:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575883.650068:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.650084:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575883.650086:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61f00. 00000100:00000001:0.0:1788575883.650091:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575883.650093:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575883.650095:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575883.650098:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575883.650100:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.650102:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=4 00000020:00000001:0.0:1788575883.650105:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575883.650107:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00010000:0.0:1788575883.650111:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650117:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575883.650121:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650127:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575883.650130:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650135:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575883.650137:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575883.650139:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650144:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575883.650145:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575883.650148:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575883.650151:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575883.650166:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575883.650168:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575883.650171:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650177:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575883.650178:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575883.650179:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575883.650180:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31600 refcount=5 00000020:00000001:0.0:1788575883.650182:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575883.650184:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536686592 : -109714172865024 : ffff9c372db31600) 00010000:00000001:0.0:1788575883.650187:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575883.650189:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575883.650194:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575883.650196:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575883.650201:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575883.650203:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575883.650204:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575883.650207:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575883.650208:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575883.650211:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575883.650212:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575883.650214:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575883.650215:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575883.650217:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31600 with handle 0x1441586093168dec from hash 00010000:00000001:0.0:1788575883.650219:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575883.650221:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575883.650222:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575883.650227:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575883.650228:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575883.650230:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575883.650232:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575883.650233:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575883.650234:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575883.650236:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575883.650239:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575883.650245:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575883.650245:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575883.650247:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575883.650249:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575883.650250:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575883.650251:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575883.650252:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575883.650254:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31600/0x1441586093168dec lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575883.650260:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692700 count: 0 00010000:00000001:0.0:1788575883.650269:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575883.650271:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575883.650274:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575883.650278:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810485248/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575883.650284:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575883.650287:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575883.650289:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575883.650291:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575883.650293:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575883.650297:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575883.650299:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575883.650301:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788575883.650304:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575883.650305:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575883.650314:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575883.650317:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575883.650319:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575887.489756:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311026463, at 4311026496 00000100:00000010:0.0:1788575887.489777:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215ae0. 00000100:00080000:0.0:1788575888.705796:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575888.705804:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575888.705806:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575888.705810:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575888.705823:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575888.705830:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16355 00000400:00000200:0.0:1788575888.705838:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575888.705844:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575888.705847:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575888.705850:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.705853:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.705856:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.705858:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575888.705862:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575888.705863:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575888.705866:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575888.705868:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575888.705870:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575888.705871:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575888.705873:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.705874:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.705875:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.705876:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575888.705878:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575888.705880:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575888.705897:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575888.705899:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575888.705901:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575888.705904:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.705905:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575888.705907:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575888.705911:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788575888.705914:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575888.705916:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575888.705918:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.705920:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575888.705922:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575888.705925:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575888.705930:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788575888.705934:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.705941:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810486016/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575888.705949:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575888.705951:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.705954:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810486016/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575888.705985:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575888.705988:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575888.705991:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575888.705992:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575888.705994:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575888.706006:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575888.706010:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16355 00000400:00000200:0.0:1788575888.706015:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575888.706020:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575888.706022:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575888.706025:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706035:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706039:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706044:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575888.706049:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575888.706051:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575888.706054:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575888.706058:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575888.706063:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575888.706069:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575888.706075:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706077:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706079:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706082:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575888.706085:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575888.706089:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706093:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575888.706094:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575888.706101:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575888.706106:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706110:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575888.706114:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575888.706117:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706121:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575888.706128:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788575888.706130:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575888.706133:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575888.706136:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706137:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575888.706139:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575888.706141:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575888.706147:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788575888.706151:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.706161:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810486144/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575888.706169:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575888.706172:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.706181:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810486144/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575888.706207:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575888.706210:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575888.706215:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575888.706217:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575888.706220:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575888.706224:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575888.706228:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16355 00000400:00000200:0.0:1788575888.706233:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575888.706235:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575888.706236:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575888.706239:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706242:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706244:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706246:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575888.706247:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575888.706248:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575888.706250:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575888.706252:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575888.706256:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575888.706258:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575888.706261:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706265:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706266:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706273:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575888.706276:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575888.706281:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706285:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575888.706288:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575888.706291:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575888.706296:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706298:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575888.706301:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575888.706304:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706305:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575888.706309:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788575888.706313:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575888.706317:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575888.706319:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706322:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575888.706324:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575888.706327:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575888.706331:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788575888.706335:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.706340:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810486272/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575888.706351:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575888.706353:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.706356:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810486272/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575888.706365:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575888.706368:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575888.706371:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575888.706374:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575888.706376:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575888.706386:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575888.706390:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16355 00000400:00000200:0.0:1788575888.706397:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575888.706399:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575888.706402:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575888.706407:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706410:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706414:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706417:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575888.706421:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575888.706423:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575888.706424:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575888.706426:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575888.706429:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575888.706431:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575888.706437:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706440:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706443:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706445:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575888.706448:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575888.706451:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706453:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575888.706454:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575888.706458:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575888.706461:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706463:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575888.706465:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575888.706469:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706470:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575888.706473:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788575888.706479:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575888.706482:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575888.706482:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706484:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575888.706486:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575888.706489:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575888.706493:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788575888.706496:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.706502:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810486400/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575888.706512:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575888.706514:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.706519:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810486400/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575888.706538:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575888.706543:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575888.706545:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575888.706547:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575888.706550:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575888.706554:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575888.706558:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16355 00000400:00000200:0.0:1788575888.706562:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575888.706567:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575888.706570:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575888.706574:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706578:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706581:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706585:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575888.706588:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575888.706590:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575888.706597:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575888.706601:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575888.706606:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575888.706610:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575888.706626:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706629:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575888.706633:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706673:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575888.706679:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575888.706685:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575888.706690:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575888.706692:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575888.706696:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575888.706701:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706705:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575888.706709:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575888.706711:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706712:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575888.706715:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788575888.706717:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575888.706719:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575888.706720:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575888.706723:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575888.706726:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575888.706730:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575888.706737:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788575888.706746:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.706759:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810486528/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575888.706781:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575888.706784:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.706793:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810486528/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575888.706820:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575888.706827:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16365) 00000100:00000040:0.0:1788575892.545754:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311031518, at 4311031552 00000100:00000010:0.0:1788575892.545772:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000100:00000040:0.0:1788575897.665788:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311036639, at 4311036672 00000100:00000010:0.0:1788575897.665809:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391360. 00000100:00000040:0.0:1788575902.785799:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311041759, at 4311041792 00000100:00000010:0.0:1788575902.785821:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391660. 00000800:00000200:0.0:1788575909.122736:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575909.255325:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311048225, at 4311048261 00000100:00000010:0.0:1788575909.255342:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391300. 00000100:00000040:0.0:1788575914.562382:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311053535, at 4311053568 00000100:00000010:0.0:1788575914.562390:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391960. 00000100:00080000:0.0:1788575915.841747:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575915.841758:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575915.841760:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575915.841764:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575915.841768:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575915.841775:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16382 00000400:00000200:0.0:1788575915.841783:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575915.841788:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575915.841791:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575915.841794:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841797:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.841799:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841802:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575915.841805:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575915.841807:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575915.841825:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575915.841829:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575915.841831:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575915.841833:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575915.841837:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841839:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.841840:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841842:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575915.841845:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575915.841848:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841851:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575915.841854:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575915.841857:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575915.841860:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.841862:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575915.841864:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575915.841870:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788575915.841873:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575915.841877:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575915.841879:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.841881:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575915.841883:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575915.841887:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575915.841894:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575915.841901:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.841910:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810489984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575915.841930:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575915.841934:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.841938:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810489984/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575915.841962:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575915.841971:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575915.841975:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575915.841976:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575915.841977:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575915.841980:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575915.841982:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16382 00000400:00000200:0.0:1788575915.841987:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575915.841990:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575915.841992:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575915.841994:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.841997:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.841998:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842000:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575915.842002:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575915.842004:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575915.842006:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575915.842007:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575915.842010:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575915.842012:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575915.842015:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842016:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842018:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842020:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575915.842022:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575915.842024:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842026:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575915.842028:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575915.842034:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575915.842036:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842038:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575915.842047:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575915.842050:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842052:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575915.842055:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575915.842058:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575915.842060:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575915.842061:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842063:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575915.842064:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575915.842066:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575915.842069:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575915.842073:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.842078:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810490112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575915.842085:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575915.842087:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.842090:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810490112/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575915.842098:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575915.842100:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575915.842103:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575915.842104:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575915.842106:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575915.842108:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575915.842110:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16382 00000400:00000200:0.0:1788575915.842115:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575915.842118:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575915.842120:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575915.842122:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842124:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842126:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842133:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575915.842135:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575915.842137:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575915.842138:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575915.842140:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575915.842142:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575915.842144:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575915.842147:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842148:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842150:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842152:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575915.842154:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575915.842157:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842159:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575915.842161:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575915.842164:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575915.842166:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842167:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575915.842169:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575915.842171:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842173:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575915.842176:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788575915.842178:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575915.842180:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575915.842182:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842183:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575915.842184:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575915.842186:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575915.842189:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d400. 00000100:00000001:0.0:1788575915.842193:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.842196:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810490240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575915.842211:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575915.842213:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.842216:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810490240/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575915.842224:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575915.842226:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575915.842229:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575915.842231:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575915.842232:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575915.842234:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575915.842236:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16382 00000400:00000200:0.0:1788575915.842239:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575915.842242:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575915.842244:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575915.842247:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842248:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842250:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842252:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575915.842254:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575915.842255:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575915.842256:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575915.842258:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575915.842272:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575915.842274:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575915.842277:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842279:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842281:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842283:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575915.842289:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575915.842292:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842294:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575915.842296:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575915.842299:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575915.842300:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842302:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575915.842304:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575915.842306:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842308:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575915.842310:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788575915.842312:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575915.842314:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575915.842316:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842318:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575915.842319:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575915.842321:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575915.842324:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575915.842355:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.842360:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810490368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575915.842367:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575915.842368:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.842372:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810490368/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575915.842379:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575915.842381:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575915.842385:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575915.842386:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575915.842388:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575915.842390:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575915.842396:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16382 00000400:00000200:0.0:1788575915.842400:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575915.842403:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575915.842405:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575915.842407:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842409:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842411:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842413:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575915.842415:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575915.842421:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575915.842423:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575915.842425:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575915.842426:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575915.842428:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575915.842431:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842433:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575915.842435:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842437:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575915.842440:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575915.842442:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575915.842444:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575915.842446:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575915.842449:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575915.842450:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842452:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575915.842454:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575915.842456:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842458:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575915.842470:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788575915.842472:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575915.842475:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575915.842480:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.842482:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575915.842483:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575915.842485:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575915.842489:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575915.842492:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.842497:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810490496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575915.842503:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575915.842505:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.842508:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810490496/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575915.842515:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575915.842519:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16392) 00000100:00000040:0.0:1788575919.681872:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311058654, at 4311058688 00000100:00000010:0.0:1788575919.681878:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000100:00000040:0.0:1788575924.801780:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311063774, at 4311063808 00000100:00000010:0.0:1788575924.801797:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215d80. 10000000:00000001:0.0:1788575925.569774:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575925.569790:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9640. 00000020:00000010:0.0:1788575925.569798:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575925.569803:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575925.569807:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575925.569813:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575925.569818:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575925.569823:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575925.569827:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575925.569832:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788575925.569837:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575925.569840:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575925.569845:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575925.569851:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575925.569865:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575925.569868:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575925.569870:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575925.569872:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575925.569874:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575925.569876:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788575925.569878:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9640. 10000000:01000000:0.0:1788575925.569880:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575925.569882:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575925.569884:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575925.569885:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575925.569887:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575925.569889:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575925.569890:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575925.569892:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575925.569893:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575925.569895:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575925.569896:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575925.569898:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575925.569899:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575927.553774:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575927.553783:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575927.553787:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575927.553790:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575927.553791:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575927.553792:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575927.553802:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575927.553805:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575927.553808:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575927.553811:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575927.553813:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575927.553830:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575927.553836:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575927.553842:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575927.553847:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575927.553853:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575927.553861:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575927.553896:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575927.553902:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575927.553906:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575927.553916:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692a00. 00010000:00000001:0.0:1788575927.553921:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575927.553925:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31000. 00000020:00000001:0.0:1788575927.553930:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575927.553932:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31000 with handle 0x1441586093168dfa to hash 00000020:00000001:0.0:1788575927.553934:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575927.553935:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575927.553938:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575927.553941:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575927.553944:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575927.553947:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575927.553956:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575927.553966:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575927.553971:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575927.553973:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575927.553979:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575927.553981:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611392 : -109714523940224 : ffff9c3718c61a80) 00000100:00000001:0.0:1788575927.553983:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575927.553984:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788575927.553989:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575927.553999:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575927.554002:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575927.554004:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575927.554018:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575927.554039:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575927.554044:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575927.554048:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575927.554051:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575927.554056:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575927.554058:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575927.554059:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575927.554061:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575927.554063:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575927.554064:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575927.554067:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575927.554069:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575927.554071:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575927.554074:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575927.554080:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810491904:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575927.554084:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575927.554086:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575927.554088:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575927.554090:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575927.554092:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575927.554097:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575927.554099:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575927.554105:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575927.554107:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575927.554109:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575927.554116:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575927.554117:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575927.554119:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=4 00000020:00000001:0.0:1788575927.554121:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575927.554123:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00010000:0.0:1788575927.554134:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554141:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575927.554145:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554152:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575927.554155:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554162:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575927.554164:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575927.554167:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554172:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575927.554174:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575927.554177:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575927.554180:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575927.554187:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31000 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575927.554193:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575927.554196:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554213:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575927.554214:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575927.554215:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575927.554229:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31000 refcount=5 00000020:00000001:0.0:1788575927.554231:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575927.554233:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536685056 : -109714172866560 : ffff9c372db31000) 00010000:00000001:0.0:1788575927.554236:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575927.554238:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575927.554243:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575927.554244:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575927.554248:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575927.554250:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575927.554251:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575927.554253:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575927.554254:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575927.554257:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575927.554258:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575927.554259:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575927.554260:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575927.554262:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31000 with handle 0x1441586093168dfa from hash 00010000:00000001:0.0:1788575927.554264:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575927.554265:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575927.554266:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575927.554266:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575927.554268:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575927.554269:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575927.554271:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575927.554271:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575927.554272:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575927.554277:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575927.554279:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575927.554283:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575927.554284:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575927.554285:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575927.554286:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575927.554287:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575927.554287:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575927.554288:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575927.554289:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31000/0x1441586093168dfa lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575927.554295:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692a00 count: 0 00010000:00000001:0.0:1788575927.554303:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575927.554304:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575927.554306:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575927.554309:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810491904/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575927.554314:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575927.554317:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575927.554318:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575927.554320:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575927.554322:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575927.554327:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575927.554329:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575927.554331:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788575927.554335:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575927.554336:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575927.554338:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575927.554340:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575927.554344:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575931.393713:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311070368, at 4311070400 00000100:00000010:0.0:1788575931.393731:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215360. 00000100:00080000:0.0:1788575932.737792:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575932.737804:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575932.737807:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575932.737810:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575932.737814:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575932.737821:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16399 00000400:00000200:0.0:1788575932.737829:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575932.737835:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575932.737838:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575932.737841:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.737844:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.737847:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.737849:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575932.737853:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575932.737855:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788575932.737857:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575932.737861:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788575932.737863:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788575932.737865:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788575932.737868:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.737870:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.737872:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.737874:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575932.737877:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788575932.737880:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575932.737882:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575932.737885:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575932.737888:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575932.737891:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.737893:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575932.737910:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575932.737915:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575932.737919:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788575932.737923:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575932.737924:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.737927:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575932.737930:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575932.737934:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575932.737942:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788575932.737949:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.737958:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810492672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575932.737967:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575932.737971:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.737975:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810492672/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575932.737996:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575932.737998:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575932.738002:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575932.738003:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575932.738005:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575932.738007:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575932.738009:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16399 00000400:00000200:0.0:1788575932.738013:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575932.738016:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575932.738018:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575932.738022:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738024:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738026:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738029:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575932.738031:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575932.738042:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788575932.738045:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575932.738047:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788575932.738051:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788575932.738054:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788575932.738071:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738082:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738088:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738094:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575932.738099:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788575932.738103:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738107:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575932.738111:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575932.738119:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575932.738123:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738125:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575932.738129:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575932.738133:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738137:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575932.738143:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788575932.738147:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575932.738151:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575932.738152:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738155:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575932.738158:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575932.738160:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575932.738165:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788575932.738183:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.738192:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810492800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575932.738213:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575932.738218:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.738230:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810492800/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575932.738245:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575932.738249:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575932.738255:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575932.738258:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575932.738260:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575932.738263:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575932.738268:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16399 00000400:00000200:0.0:1788575932.738275:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575932.738280:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575932.738285:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575932.738292:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738297:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738303:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738307:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575932.738312:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575932.738315:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788575932.738317:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575932.738319:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788575932.738324:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788575932.738329:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788575932.738334:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738338:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738342:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738346:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575932.738350:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788575932.738359:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738363:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575932.738372:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575932.738375:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575932.738378:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738381:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575932.738385:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575932.738389:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738393:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575932.738399:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788575932.738402:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788575932.738408:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575932.738411:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738415:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575932.738419:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575932.738421:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575932.738427:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788575932.738433:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.738441:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810492928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575932.738458:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575932.738462:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.738470:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810492928/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575932.738486:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575932.738490:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575932.738497:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575932.738499:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575932.738502:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575932.738506:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575932.738511:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16399 00000400:00000200:0.0:1788575932.738518:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575932.738524:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575932.738528:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575932.738539:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738544:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738548:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738553:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575932.738556:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575932.738558:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788575932.738562:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575932.738566:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788575932.738571:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788575932.738576:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788575932.738582:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738587:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738592:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738598:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575932.738602:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788575932.738608:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738611:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575932.738613:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575932.738617:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575932.738622:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738626:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575932.738628:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575932.738630:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738632:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575932.738673:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788575932.738679:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788575932.738686:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575932.738690:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738694:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575932.738698:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575932.738701:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575932.738710:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788575932.738721:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.738728:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810493056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575932.738743:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575932.738748:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.738758:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810493056/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575932.738785:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575932.738790:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575932.738795:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575932.738797:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575932.738801:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575932.738805:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575932.738810:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16399 00000400:00000200:0.0:1788575932.738818:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788575932.738825:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575932.738829:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575932.738837:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738841:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738847:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738852:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788575932.738856:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575932.738859:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788575932.738863:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575932.738867:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788575932.738872:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788575932.738876:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788575932.738883:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738888:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575932.738898:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738905:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788575932.738911:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788575932.738915:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788575932.738921:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575932.738926:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575932.738931:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575932.738936:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738941:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575932.738945:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575932.738951:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738955:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575932.738962:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788575932.738967:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788575932.738973:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575932.738977:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575932.738983:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575932.738987:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575932.738991:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575932.738999:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788575932.739006:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.739014:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810493184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575932.739029:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575932.739033:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.739039:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810493184/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575932.739056:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.739063:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16409) 00000100:00000040:0.0:1788575936.577780:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311075550, at 4311075584 00000100:00000010:0.0:1788575936.577797:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215060. 00000100:00000040:0.0:1788575941.697732:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311080671, at 4311080704 00000100:00000010:0.0:1788575941.697763:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215300. 00000100:00000040:0.0:1788575946.817763:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311085791, at 4311085824 00000100:00000010:0.0:1788575946.817784:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215c60. 00000800:00000200:0.0:1788575947.009873:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 10000000:00000001:0.0:1788575947.585743:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575947.585760:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788575947.585766:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575947.585771:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575947.585774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575947.585778:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575947.585781:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575947.585784:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575947.585788:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788575947.585791:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ae00. 10000000:00000040:0.0:1788575947.585795:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575947.585797:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575947.585800:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b600. 00000040:00000010:0.0:1788575947.585806:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788575947.585808:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575947.585810:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575947.585812:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575947.585815:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788575947.585816:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788575947.585818:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788575947.585820:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788575947.585822:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575947.585824:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575947.585826:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575947.585827:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575947.585828:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575947.585830:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575947.585832:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575947.585834:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575947.585846:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575947.585848:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575947.585850:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575947.585851:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575947.585853:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575948.929755:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575948.929762:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575948.929766:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575948.929767:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575948.929769:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575948.929770:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575948.929773:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575948.929774:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575948.929776:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575948.929778:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575948.929779:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575948.929788:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575948.929792:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575948.929796:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575948.929799:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575948.929803:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575948.929808:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575948.929813:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575948.929816:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575948.929818:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575948.929825:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788575948.929828:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575948.929830:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788575948.929833:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575948.929836:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168e01 to hash 00000020:00000001:0.0:1788575948.929837:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575948.929838:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575948.929840:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575948.929842:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575948.929845:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575948.929847:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575948.929864:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575948.929873:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575948.929878:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575948.929879:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575948.929884:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575948.929886:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611392 : -109714523940224 : ffff9c3718c61a80) 00000100:00000001:0.0:1788575948.929888:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575948.929889:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788575948.929892:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575948.929895:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575948.929896:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575948.929897:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575948.929904:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575948.929910:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575948.929913:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575948.929917:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575948.929919:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575948.929923:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575948.929924:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575948.929925:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575948.929927:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575948.929928:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575948.929930:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575948.929935:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575948.929937:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575948.929938:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575948.929941:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575948.929946:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810495232:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575948.929949:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575948.929951:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575948.929952:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575948.929954:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575948.929956:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575948.929960:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575948.929962:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575948.929966:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575948.929967:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575948.929969:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575948.929971:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575948.929972:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575948.929974:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788575948.929975:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575948.929977:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788575948.929979:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.929982:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575948.929985:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.929988:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575948.929992:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.929996:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575948.929997:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575948.929999:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.930002:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575948.930003:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575948.930005:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575948.930006:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575948.930010:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575948.930011:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575948.930013:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.930016:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575948.930017:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575948.930017:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575948.930018:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788575948.930031:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575948.930033:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575948.930035:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575948.930037:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575948.930041:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575948.930042:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575948.930046:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575948.930051:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575948.930052:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575948.930054:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575948.930055:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575948.930057:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575948.930058:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575948.930059:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575948.930060:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575948.930062:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168e01 from hash 00010000:00000001:0.0:1788575948.930064:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575948.930065:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575948.930065:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575948.930066:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575948.930067:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575948.930068:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575948.930069:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575948.930070:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575948.930071:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575948.930072:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575948.930073:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575948.930076:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575948.930077:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575948.930078:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575948.930079:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575948.930080:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575948.930081:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575948.930081:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575948.930083:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e01 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575948.930087:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788575948.930093:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575948.930094:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575948.930095:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575948.930100:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810495232/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575948.930111:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575948.930113:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575948.930114:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575948.930116:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575948.930118:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575948.930121:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575948.930122:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575948.930123:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788575948.930126:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575948.930126:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575948.930128:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575948.930130:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575948.930133:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575952.769760:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311091743, at 4311091776 00000100:00000010:0.0:1788575952.769775:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 00000100:00000040:0.0:1788575958.081734:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311097058, at 4311097088 00000100:00000010:0.0:1788575958.081753:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00000040:0.0:1788575963.201758:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311102175, at 4311102208 00000100:00000010:0.0:1788575963.201774:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322153c0. 00000100:00000040:0.0:1788575968.321748:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311107295, at 4311107328 00000100:00000010:0.0:1788575968.321765:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 10000000:00000001:0.0:1788575969.089743:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575969.089757:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788575969.089765:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575969.089770:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575969.089774:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575969.089779:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575969.089784:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575969.089788:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575969.089793:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575969.089807:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788575969.089813:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575969.089815:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575969.089821:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4ae00. 00000040:00000010:0.0:1788575969.089827:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575969.089831:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575969.089863:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b000. 80000000:00000010:0.0:1788575969.089867:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575969.089870:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4ba00. 20000000:00000010:0.0:1788575969.089873:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788575969.089875:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4bc00. 10000000:00000010:0.0:1788575969.089877:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788575969.089881:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575969.089884:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575969.089887:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575969.089888:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575969.089891:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575969.089894:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575969.089896:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575969.089899:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575969.089901:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575969.089903:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575969.089906:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575969.089908:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575969.089910:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575971.073753:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575971.073759:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575971.073763:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575971.073765:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575971.073766:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575971.073767:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575971.073769:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575971.073771:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575971.073772:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575971.073775:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575971.073776:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575971.073795:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575971.073799:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575971.073802:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575971.073805:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575971.073809:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575971.073829:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4bc00. 00000100:00000001:0.0:1788575971.073834:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575971.073838:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575971.073840:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575971.073848:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788575971.073852:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575971.073855:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788575971.073859:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575971.073862:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168e08 to hash 00000020:00000001:0.0:1788575971.073864:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575971.073866:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575971.073869:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575971.073872:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575971.073875:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575971.073879:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575971.073888:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575971.073899:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575971.073905:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575971.073906:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575971.073918:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575971.073921:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611392 : -109714523940224 : ffff9c3718c61a80) 00000100:00000001:0.0:1788575971.073924:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575971.073926:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788575971.073933:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575971.073936:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575971.073938:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575971.073939:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575971.073947:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575971.073954:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575971.073956:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575971.073960:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575971.073963:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575971.073967:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575971.073969:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575971.073970:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575971.073971:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575971.073973:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575971.073974:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575971.073977:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575971.073978:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575971.073980:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575971.073982:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575971.073987:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810498560:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575971.073990:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575971.073992:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575971.073994:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575971.073995:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575971.073997:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575971.074008:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575971.074010:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575971.074014:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575971.074015:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575971.074016:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575971.074019:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575971.074020:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575971.074022:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788575971.074023:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575971.074025:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788575971.074027:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074032:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575971.074034:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074038:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575971.074039:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074043:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575971.074044:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575971.074045:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074048:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575971.074049:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575971.074051:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575971.074053:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575971.074066:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575971.074068:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575971.074071:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074076:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575971.074077:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575971.074078:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575971.074079:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788575971.074081:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575971.074083:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575971.074085:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575971.074088:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575971.074094:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575971.074096:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575971.074101:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575971.074104:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575971.074105:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575971.074108:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575971.074109:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575971.074111:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575971.074113:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575971.074114:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575971.074115:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575971.074117:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168e08 from hash 00010000:00000001:0.0:1788575971.074118:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575971.074119:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575971.074120:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575971.074124:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575971.074126:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575971.074128:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575971.074130:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575971.074131:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575971.074132:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575971.074134:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575971.074137:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575971.074142:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575971.074143:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575971.074145:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575971.074146:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575971.074148:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575971.074149:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575971.074150:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575971.074152:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e08 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575971.074159:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788575971.074167:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575971.074169:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575971.074171:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575971.074176:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810498560/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575971.074182:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575971.074184:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575971.074185:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575971.074187:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575971.074190:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4bc00. 02000000:00000001:0.0:1788575971.074194:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575971.074196:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575971.074197:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788575971.074200:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575971.074201:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575971.074208:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575971.074210:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575971.074213:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575974.913722:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311113887, at 4311113920 00000100:00000010:0.0:1788575974.913734:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00000040:0.0:1788575980.097795:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311119070, at 4311119104 00000100:00000010:0.0:1788575980.097809:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322152a0. 00000100:00000040:0.0:1788575985.217728:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311124191, at 4311124224 00000100:00000010:0.0:1788575985.217751:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215b40. 00000800:00000200:0.0:1788575985.921947:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1788575990.337741:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311129310, at 4311129344 00000100:00000010:0.0:1788575990.337760:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215e40. 10000000:00000001:0.0:1788575991.105765:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788575991.105780:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788575991.105788:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788575991.105793:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788575991.105797:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575991.105802:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575991.105807:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575991.105811:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575991.105816:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788575991.105820:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788575991.105824:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788575991.105827:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788575991.105832:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788575991.105838:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788575991.105841:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788575991.105844:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788575991.105847:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788575991.105850:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788575991.105852:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788575991.105855:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788575991.105869:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788575991.105872:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788575991.105875:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788575991.105877:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788575991.105879:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788575991.105881:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788575991.105884:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788575991.105886:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575991.105889:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788575991.105890:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575991.105893:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788575991.105895:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788575991.105897:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575991.105899:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788575993.025764:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788575993.025772:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788575993.025778:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575993.025782:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788575993.025783:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788575993.025785:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788575993.025788:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788575993.025790:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788575993.025794:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788575993.025800:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788575993.025802:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788575993.025812:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788575993.025826:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788575993.025831:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575993.025834:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575993.025841:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575993.025850:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788575993.025857:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575993.025862:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788575993.025864:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788575993.025870:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788575993.025873:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788575993.025877:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788575993.025886:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788575993.025888:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168e0f to hash 00000020:00000001:0.0:1788575993.025892:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788575993.025895:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575993.025899:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575993.025902:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575993.025905:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575993.025908:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575993.025914:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575993.025922:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788575993.025926:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788575993.025927:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788575993.025932:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575993.025934:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611392 : -109714523940224 : ffff9c3718c61a80) 00000100:00000001:0.0:1788575993.025935:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575993.025936:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788575993.025940:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575993.025943:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575993.025945:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788575993.025946:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575993.025954:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575993.025960:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788575993.025962:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575993.025967:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575993.025972:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575993.025976:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575993.025978:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575993.025979:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575993.025980:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575993.025982:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575993.025984:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788575993.025986:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575993.025988:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575993.025989:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575993.025992:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575993.025996:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810501888:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788575993.026000:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575993.026001:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575993.026003:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788575993.026005:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575993.026007:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575993.026011:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575993.026012:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788575993.026018:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788575993.026019:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788575993.026021:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788575993.026024:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575993.026026:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575993.026028:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788575993.026030:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575993.026032:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788575993.026036:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026059:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788575993.026063:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026067:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788575993.026068:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026072:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575993.026073:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788575993.026074:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026077:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575993.026078:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788575993.026080:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788575993.026081:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788575993.026085:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788575993.026086:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575993.026088:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026091:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788575993.026092:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788575993.026092:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575993.026093:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788575993.026094:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575993.026095:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788575993.026100:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788575993.026102:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788575993.026106:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788575993.026108:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788575993.026111:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788575993.026112:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788575993.026113:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575993.026115:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788575993.026116:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788575993.026118:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788575993.026119:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788575993.026120:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788575993.026122:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788575993.026123:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168e0f from hash 00010000:00000001:0.0:1788575993.026125:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788575993.026126:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575993.026126:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575993.026127:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788575993.026128:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788575993.026129:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788575993.026130:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575993.026131:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575993.026132:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788575993.026133:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575993.026135:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788575993.026139:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575993.026140:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575993.026141:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788575993.026142:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788575993.026147:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788575993.026148:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575993.026149:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788575993.026151:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e0f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788575993.026158:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788575993.026166:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788575993.026167:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788575993.026170:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575993.026174:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810501888/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575993.026181:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788575993.026183:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575993.026184:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788575993.026185:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575993.026187:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788575993.026189:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575993.026191:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575993.026192:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788575993.026195:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575993.026195:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788575993.026197:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788575993.026198:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788575993.026200:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788575996.865759:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311135839, at 4311135872 00000100:00000010:0.0:1788575996.865774:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322155a0. 00000100:00000040:0.0:1788576002.113760:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311141086, at 4311141120 00000100:00000010:0.0:1788576002.113779:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215c00. 00000100:00000040:0.0:1788576007.233758:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311146206, at 4311146240 00000100:00000010:0.0:1788576007.233774:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215e40. 00000100:00000040:0.0:1788576012.353757:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311151326, at 4311151360 00000100:00000010:0.0:1788576012.353774:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 10000000:00000001:0.0:1788576013.121752:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576013.121763:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788576013.121769:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788576013.121773:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788576013.121776:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788576013.121779:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576013.121783:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576013.121786:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576013.121790:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576013.121793:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b000. 10000000:00000040:0.0:1788576013.121797:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576013.121798:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576013.121802:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4bc00. 00000040:00000010:0.0:1788576013.121807:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788576013.121809:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576013.121812:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576013.121814:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576013.121816:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576013.121817:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b000. 00000020:00000010:0.0:1788576013.121819:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788576013.121821:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788576013.121823:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576013.121825:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576013.121827:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576013.121828:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576013.121829:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576013.121831:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576013.121833:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576013.121835:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576013.121835:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576013.121837:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576013.121839:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576013.121840:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576013.121842:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576014.593737:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576014.593745:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576014.593750:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576014.593752:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576014.593754:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576014.593756:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576014.593759:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576014.593761:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576014.593763:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576014.593766:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576014.593768:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576014.593778:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576014.593782:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576014.593786:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576014.593790:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576014.593794:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576014.593801:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788576014.593807:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576014.593811:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576014.593814:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576014.593821:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788576014.593824:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576014.593827:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788576014.593832:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576014.593834:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168e16 to hash 00000020:00000001:0.0:1788576014.593836:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576014.593838:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576014.593841:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576014.593845:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576014.593848:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576014.593852:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576014.593863:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576014.593891:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576014.593898:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576014.593900:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576014.593906:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788576014.593908:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611392 : -109714523940224 : ffff9c3718c61a80) 00000100:00000001:0.0:1788576014.593910:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576014.593912:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576014.593916:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576014.593934:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576014.593937:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576014.593938:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576014.593948:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576014.593957:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576014.593961:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576014.593968:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576014.593972:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576014.593979:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576014.593982:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576014.593983:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576014.593986:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576014.593988:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576014.593990:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576014.593994:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.593996:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576014.593998:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576014.594037:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576014.594046:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810505216:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576014.594052:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.594055:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576014.594057:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576014.594059:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576014.594063:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576014.594070:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576014.594072:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61a80. 00000100:00000001:0.0:1788576014.594078:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576014.594079:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576014.594082:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576014.594085:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576014.594087:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576014.594089:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788576014.594091:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576014.594094:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788576014.594097:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594104:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576014.594108:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594122:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576014.594125:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594140:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576014.594142:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576014.594145:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594150:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576014.594152:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576014.594155:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576014.594158:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576014.594164:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576014.594167:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576014.594170:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594174:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576014.594176:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576014.594177:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576014.594179:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788576014.594181:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576014.594183:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576014.594185:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576014.594188:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576014.594193:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576014.594195:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576014.594201:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576014.594206:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576014.594208:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576014.594211:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576014.594213:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576014.594216:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576014.594217:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576014.594219:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576014.594221:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576014.594223:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168e16 from hash 00010000:00000001:0.0:1788576014.594226:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576014.594227:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576014.594228:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576014.594229:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576014.594230:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576014.594232:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576014.594234:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576014.594235:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576014.594237:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576014.594238:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576014.594251:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576014.594257:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576014.594258:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576014.594259:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576014.594261:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576014.594262:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576014.594263:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576014.594265:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576014.594267:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e16 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576014.594274:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788576014.594282:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576014.594283:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576014.594286:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576014.594296:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810505216/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576014.594311:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576014.594313:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576014.594315:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576014.594317:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576014.594320:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788576014.594323:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576014.594325:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576014.594327:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576014.594330:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576014.594332:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576014.594334:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576014.594336:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576014.594339:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576018.433776:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311157407, at 4311157440 00000100:00000010:0.0:1788576018.433793:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00000040:0.0:1788576023.617765:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311162595, at 4311162624 00000100:00000010:0.0:1788576023.617781:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000800:00000200:0.0:1788576023.810020:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576023.810024:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576023.810033:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576023.810036:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576023.810037:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576028.737782:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311167711, at 4311167744 00000100:00000010:0.0:1788576028.737798:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000100:00000040:0.0:1788576033.858215:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311172830, at 4311172864 00000100:00000010:0.0:1788576033.858229:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 10000000:00000001:0.0:1788576034.626148:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576034.626164:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788576034.626172:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788576034.626178:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576034.626192:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788576034.626211:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576034.626217:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576034.626221:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576034.626227:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788576034.626232:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4bc00. 10000000:00000040:0.0:1788576034.626237:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576034.626240:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576034.626247:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576034.626254:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4be00. 00000020:00000010:0.0:1788576034.626257:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576034.626261:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576034.626264:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576034.626267:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b600. 20000000:00000010:0.0:1788576034.626270:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4bc00. 00000020:00000010:0.0:1788576034.626273:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788576034.626275:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788576034.626278:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576034.626282:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576034.626284:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576034.626286:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576034.626288:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576034.626292:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576034.626294:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576034.626298:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576034.626299:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576034.626302:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576034.626304:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576034.626305:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576034.626308:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576036.225858:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576036.225867:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576036.225872:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576036.225874:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576036.225876:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576036.225878:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576036.225895:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576036.225897:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576036.225899:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576036.225902:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576036.225904:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576036.225912:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576036.225916:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576036.225921:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576036.225924:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576036.225929:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576036.225936:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788576036.225943:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576036.225947:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576036.225949:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576036.225956:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788576036.225960:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576036.225962:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30600. 00000020:00000001:0.0:1788576036.225966:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576036.225969:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30600 with handle 0x1441586093168e1d to hash 00000020:00000001:0.0:1788576036.225971:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576036.225972:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576036.225975:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576036.225979:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576036.225982:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576036.225986:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576036.225995:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576036.226005:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576036.226011:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576036.226018:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576036.226023:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576036.226025:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788576036.226028:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576036.226029:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576036.226034:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576036.226038:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576036.226040:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576036.226042:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576036.226051:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576036.226059:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576036.226062:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576036.226069:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576036.226072:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576036.226078:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576036.226080:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576036.226082:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576036.226084:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576036.226087:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576036.226089:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576036.226092:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.226094:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576036.226097:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576036.226100:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576036.226108:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810508544:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576036.226124:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.226133:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576036.226136:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576036.226138:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576036.226143:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576036.226150:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576036.226152:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576036.226158:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576036.226159:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576036.226162:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576036.226165:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576036.226167:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576036.226169:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=4 00000020:00000001:0.0:1788576036.226171:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576036.226174:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00010000:0.0:1788576036.226177:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226183:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576036.226187:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226192:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576036.226195:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226199:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576036.226201:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576036.226203:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226208:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576036.226215:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576036.226217:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576036.226220:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576036.226226:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30600 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576036.226228:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576036.226230:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226235:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576036.226237:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576036.226238:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576036.226239:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30600 refcount=5 00000020:00000001:0.0:1788576036.226241:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576036.226243:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536682496 : -109714172869120 : ffff9c372db30600) 00010000:00000001:0.0:1788576036.226246:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576036.226248:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576036.226253:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576036.226255:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576036.226260:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576036.226262:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576036.226264:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576036.226266:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576036.226268:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576036.226271:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576036.226272:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576036.226274:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576036.226275:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576036.226277:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30600 with handle 0x1441586093168e1d from hash 00010000:00000001:0.0:1788576036.226293:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576036.226295:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576036.226296:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576036.226296:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576036.226298:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576036.226299:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576036.226302:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576036.226303:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576036.226304:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576036.226306:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576036.226309:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576036.226313:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576036.226314:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576036.226315:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576036.226317:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576036.226318:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576036.226319:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576036.226320:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576036.226322:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30600/0x1441586093168e1d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576036.226328:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788576036.226336:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576036.226337:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576036.226340:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576036.226347:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810508544/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576036.226353:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576036.226356:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576036.226358:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576036.226359:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576036.226362:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788576036.226365:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576036.226367:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576036.226382:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576036.226385:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576036.226387:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576036.226388:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576036.226390:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576036.226393:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576040.066884:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311179039, at 4311179073 00000100:00000010:0.0:1788576040.066900:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322159c0. 00000100:00000040:0.0:1788576045.121830:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311184095, at 4311184128 00000100:00000010:0.0:1788576045.121837:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322150c0. 00000400:00000100:0.0:1788576049.474107:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.148:988 00000400:00000100:0.0:1788576049.474162:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.148@tcp at host 192.168.202.148 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788576049.474172:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576049.474175:0:2390:0:(router.c:1791:lnet_notify()) 192.168.202.48@tcp notifying 192.168.202.148@tcp: down 00000400:00000200:0.0:1788576049.474180:0:2390:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576049.474184:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788576049.474193:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788576049.474198:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498c00. 00000400:00000200:0.0:1788576049.474204:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788576049.474209:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788576049.474215:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1a18 00000400:00000010:0.0:1788576049.474217:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1a18. 00000100:00000001:0.0:1788576049.474222:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576049.474231:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6a300 x1875440810509952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474258:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576049.474263:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6a300 x1875440810509952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474270:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576049.474272:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788576049.474281:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788576049.474283:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498e00. 00000400:00000200:0.0:1788576049.474302:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788576049.474306:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788576049.474310:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c15d8 00000400:00000010:0.0:1788576049.474312:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c15d8. 00000100:00000001:0.0:1788576049.474314:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576049.474318:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6b480 x1875440810510080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474326:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576049.474329:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b480 x1875440810510080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474335:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576049.474336:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788576049.474340:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788576049.474348:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498600. 00000400:00000200:0.0:1788576049.474351:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788576049.474354:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788576049.474357:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c13b8 00000400:00000010:0.0:1788576049.474358:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c13b8. 00000100:00000001:0.0:1788576049.474360:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576049.474364:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6b800 x1875440810510208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474371:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576049.474374:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6b800 x1875440810510208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474380:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576049.474381:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788576049.474386:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788576049.474388:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498f00. 00000400:00000200:0.0:1788576049.474390:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788576049.474392:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788576049.474395:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1908 00000400:00000010:0.0:1788576049.474403:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1908. 00000100:00000001:0.0:1788576049.474406:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576049.474410:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed68380 x1875440810510336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474417:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576049.474420:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed68380 x1875440810510336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474432:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576049.474434:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788576049.474439:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.148@tcp 00000800:00000010:0.0:1788576049.474441:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c3611498300. 00000400:00000200:0.0:1788576049.474443:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788576049.474446:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788576049.474448:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c37321c1550 00000400:00000010:0.0:1788576049.474450:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c37321c1550. 00000100:00000001:0.0:1788576049.474452:0:2390:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576049.474456:0:2390:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff9c371ed6aa00 x1875440810510464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474461:0:2390:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576049.474464:0:2390:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ed6aa00 x1875440810510464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576049.474469:0:2390:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576049.474471:0:2390:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000040:0.0:1788576050.241745:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311189214, at 4311189248 00000100:00000010:0.0:1788576050.241759:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215f00. 00000100:00000040:0.0:1788576055.361740:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311194335, at 4311194368 00000100:00000010:0.0:1788576055.361757:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215540. 10000000:00000001:0.0:1788576056.641760:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576056.641770:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9000. 00000020:00000010:0.0:1788576056.641778:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4be00. 00000020:00000010:0.0:1788576056.641801:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576056.641813:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788576056.641820:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576056.641828:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788576056.641836:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576056.641842:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788576056.641847:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ae00. 10000000:00000040:0.0:1788576056.641852:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576056.641856:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576056.641862:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576056.641869:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788576056.641874:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576056.641880:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788576056.641883:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576056.641887:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4bc00. 20000000:00000010:0.0:1788576056.641889:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788576056.641893:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4be00. 10000000:00000010:0.0:1788576056.641895:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9000. 10000000:01000000:0.0:1788576056.641900:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576056.641903:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576056.641906:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576056.641907:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576056.641909:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576056.641912:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576056.641914:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576056.641918:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576056.641921:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576056.641925:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576056.641929:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576056.641932:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576056.641941:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576058.497944:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576058.497955:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576058.497961:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576058.497964:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576058.497965:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576058.497987:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576058.497991:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576058.497992:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576058.497995:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576058.497999:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576058.498001:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576058.498010:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576058.498016:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576058.498022:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576058.498026:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576058.498033:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576058.498043:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4be00. 00000100:00000001:0.0:1788576058.498051:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576058.498056:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576058.498059:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576058.498068:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788576058.498073:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576058.498077:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788576058.498082:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576058.498085:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x1441586093168e24 to hash 00000020:00000001:0.0:1788576058.498088:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576058.498090:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576058.498093:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576058.498097:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576058.498101:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576058.498106:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576058.498117:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576058.498131:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576058.498137:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576058.498143:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576058.498151:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576058.498153:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788576058.498157:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576058.498158:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576058.498163:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576058.498169:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576058.498171:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576058.498173:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576058.498186:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576058.498196:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576058.498201:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576058.498208:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576058.498212:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576058.498219:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576058.498221:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576058.498223:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576058.498226:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576058.498229:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576058.498232:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576058.498236:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576058.498239:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576058.498241:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576058.498246:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576058.498254:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810511872:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576058.498260:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576058.498266:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576058.498269:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576058.498271:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576058.498275:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576058.498283:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576058.498285:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576058.498292:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576058.498293:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576058.498296:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576058.498299:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576058.498301:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576058.498304:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=4 00000020:00000001:0.0:1788576058.498306:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576058.498309:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00010000:0.0:1788576058.498313:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498320:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576058.498324:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498330:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576058.498333:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498339:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576058.498341:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576058.498344:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498351:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576058.498355:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576058.498358:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576058.498361:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576058.498368:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31e00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576058.498371:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576058.498373:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498380:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576058.498381:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576058.498383:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576058.498384:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=5 00000020:00000001:0.0:1788576058.498386:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576058.498388:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576058.498391:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576058.498394:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576058.498399:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576058.498402:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576058.498407:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576058.498409:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576058.498411:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576058.498414:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576058.498416:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576058.498419:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576058.498420:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576058.498422:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576058.498423:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576058.498426:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31e00 with handle 0x1441586093168e24 from hash 00010000:00000001:0.0:1788576058.498431:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576058.498433:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576058.498434:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576058.498435:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576058.498437:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576058.498439:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576058.498441:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576058.498442:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576058.498444:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576058.498446:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576058.498448:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576058.498453:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576058.498455:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576058.498456:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576058.498458:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576058.498459:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576058.498461:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576058.498462:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576058.498465:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e24 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576058.498472:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788576058.498482:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576058.498484:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576058.498486:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576058.498490:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810511872/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576058.498497:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576058.498501:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576058.498502:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576058.498504:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576058.498507:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4be00. 02000000:00000001:0.0:1788576058.498511:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576058.498514:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576058.498519:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576058.498523:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576058.498524:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576058.498526:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576058.498529:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576058.498532:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576062.337996:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311201311, at 4311201344 00000100:00000010:0.0:1788576062.338002:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215a80. 00000800:00000200:0.0:1788576062.721951:0:2396:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576062.721957:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576062.721969:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576062.721973:0:2396:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576062.721975:0:2396:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576067.650661:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311206622, at 4311206657 00000100:00000010:0.0:1788576067.650670:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 00000100:00000040:0.0:1788576072.769799:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311211742, at 4311211776 00000100:00000010:0.0:1788576072.769815:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215480. 00000100:00000040:0.0:1788576077.889764:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311216863, at 4311216896 00000100:00000010:0.0:1788576077.889779:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3732215360. 10000000:00000001:0.0:1788576078.658498:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576078.658514:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a9500. 00000020:00000010:0.0:1788576078.658522:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788576078.658527:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576078.658533:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788576078.658537:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576078.658543:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788576078.658547:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576078.658551:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576078.658556:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985dc00. 10000000:00000040:0.0:1788576078.658560:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576078.658563:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576078.658578:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4b000. 00000040:00000010:0.0:1788576078.658599:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4b600. 00000020:00000010:0.0:1788576078.658602:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576078.658606:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788576078.658609:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576078.658612:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576078.658615:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985dc00. 00000020:00000010:0.0:1788576078.658619:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ba00. 10000000:00000010:0.0:1788576078.658621:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a9500. 10000000:01000000:0.0:1788576078.658625:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576078.658628:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576078.658631:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576078.658632:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576078.658668:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576078.658672:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576078.658674:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576078.658678:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576078.658680:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576078.658682:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576078.658685:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576078.658686:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576078.658689:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576080.321934:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576080.321942:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576080.321946:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576080.321949:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576080.321951:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576080.321952:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576080.321955:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576080.321957:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576080.321960:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576080.321964:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576080.321965:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576080.321977:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576080.321981:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576080.321986:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576080.321989:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576080.322006:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576080.322014:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985dc00. 00000100:00000001:0.0:1788576080.322020:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576080.322025:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576080.322028:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576080.322036:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788576080.322040:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576080.322044:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31e00. 00000020:00000001:0.0:1788576080.322048:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576080.322051:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31e00 with handle 0x1441586093168e2b to hash 00000020:00000001:0.0:1788576080.322053:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576080.322054:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576080.322057:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576080.322061:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576080.322064:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576080.322068:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576080.322078:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576080.322089:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576080.322094:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576080.322096:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576080.322101:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576080.322103:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788576080.322105:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576080.322107:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576080.322111:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576080.322115:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576080.322117:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576080.322118:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576080.322133:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576080.322142:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576080.322146:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576080.322154:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576080.322158:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576080.322164:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576080.322167:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576080.322169:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576080.322171:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576080.322173:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576080.322175:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576080.322178:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.322180:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576080.322182:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576080.322186:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576080.322194:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810515200:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576080.322199:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.322201:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576080.322203:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576080.322205:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576080.322208:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576080.322437:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576080.322442:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576080.322448:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576080.322450:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576080.322458:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576080.322461:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576080.322463:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576080.322466:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=4 00000020:00000001:0.0:1788576080.322468:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576080.322471:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00010000:0.0:1788576080.322474:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322480:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576080.322484:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322489:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576080.322492:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322498:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576080.322499:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576080.322502:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322508:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576080.322510:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576080.322512:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576080.322515:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576080.322521:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31e00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576080.322523:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576080.322526:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322542:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576080.322544:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576080.322545:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576080.322547:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31e00 refcount=5 00000020:00000001:0.0:1788576080.322549:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576080.322551:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536688640 : -109714172862976 : ffff9c372db31e00) 00010000:00000001:0.0:1788576080.322554:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576080.322557:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576080.322563:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576080.322565:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576080.322572:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576080.322574:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576080.322576:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576080.322579:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576080.322581:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576080.322583:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576080.322585:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576080.322587:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576080.322588:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576080.322590:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31e00 with handle 0x1441586093168e2b from hash 00010000:00000001:0.0:1788576080.322593:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576080.322594:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576080.322595:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576080.322596:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576080.322597:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576080.322599:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576080.322602:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576080.322603:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576080.322604:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576080.322611:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576080.322614:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576080.322619:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576080.322620:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576080.322622:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576080.322624:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576080.322626:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576080.322627:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576080.322628:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576080.322630:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31e00/0x1441586093168e2b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576080.322713:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788576080.322726:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576080.322728:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576080.322731:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576080.322736:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810515200/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576080.322742:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576080.322745:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576080.322747:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576080.322749:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576080.322752:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985dc00. 02000000:00000001:0.0:1788576080.322757:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576080.322758:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576080.322760:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576080.322763:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576080.322764:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576080.322765:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576080.322768:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576080.322770:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576084.161751:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311223136, at 4311223168 00000100:00000010:0.0:1788576084.161765:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37322151e0. 00000100:00000040:0.0:1788576089.665737:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311228638, at 4311228672 00000100:00000010:0.0:1788576089.665754:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788576094.785784:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311233759, at 4311233792 00000100:00000010:0.0:1788576094.785798:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00000040:0.0:1788576099.905942:0:267976:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311238879, at 4311238912 00000100:00000010:0.0:1788576099.905949:0:267976:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391840. 00000800:00000200:0.0:1788576100.611180:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576100.611196:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576100.611208:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576100.611212:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576100.611215:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576100.673783:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576100.673798:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a94c0. 00000020:00000010:0.0:1788576100.673806:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371985d600. 00000020:00000010:0.0:1788576100.673814:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788576100.673818:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371985dc00. 00000020:00000010:0.0:1788576100.673824:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576100.673830:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576100.673846:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576100.673851:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576100.673855:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4ae00. 10000000:00000040:0.0:1788576100.673860:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576100.673862:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576100.673867:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371985cc00. 00000040:00000010:0.0:1788576100.673875:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371985dc00. 00000020:00000010:0.0:1788576100.673879:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576100.673883:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ba00. 80000000:00000010:0.0:1788576100.673886:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576100.673889:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4be00. 20000000:00000010:0.0:1788576100.673892:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788576100.673904:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371985d600. 10000000:00000010:0.0:1788576100.673908:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a94c0. 10000000:01000000:0.0:1788576100.673925:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576100.673928:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576100.673930:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576100.673932:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576100.673934:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576100.673937:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576100.673939:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576100.673942:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576100.673944:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576100.673946:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576100.673948:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576100.673949:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576100.673952:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576102.273832:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576102.273841:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576102.273847:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576102.273849:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576102.273851:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576102.273853:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576102.273856:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576102.273858:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576102.273860:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576102.273864:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576102.273867:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576102.273879:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576102.273885:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576102.273890:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576102.273894:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576102.273912:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576102.273922:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4ae00. 00000100:00000001:0.0:1788576102.273928:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576102.273933:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576102.273936:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576102.273944:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692200. 00010000:00000001:0.0:1788576102.273958:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576102.273962:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30200. 00000020:00000001:0.0:1788576102.273973:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576102.273995:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30200 with handle 0x1441586093168e32 to hash 00000020:00000001:0.0:1788576102.273998:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576102.274000:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576102.274003:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576102.274007:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576102.274010:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576102.274015:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576102.274026:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576102.274038:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576102.274044:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576102.274046:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576102.274052:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576102.274055:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610624 : -109714523940992 : ffff9c3718c61780) 00000100:00000001:0.0:1788576102.274058:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576102.274059:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576102.274064:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576102.274068:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576102.274070:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576102.274072:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576102.274082:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576102.274091:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576102.274095:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576102.274107:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576102.274111:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576102.274123:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576102.274126:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576102.274128:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576102.274131:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576102.274133:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576102.274135:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576102.274139:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576102.274141:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576102.274143:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576102.274147:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576102.274155:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810518528:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576102.274161:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576102.274163:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576102.274165:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576102.274167:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576102.274171:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576102.274178:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576102.274181:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61780. 00000100:00000001:0.0:1788576102.274187:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576102.274188:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576102.274191:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576102.274194:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576102.274196:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576102.274198:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=4 00000020:00000001:0.0:1788576102.274201:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576102.274204:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00010000:0.0:1788576102.274208:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274218:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576102.274222:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274228:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576102.274231:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274237:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576102.274238:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576102.274240:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274245:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576102.274247:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576102.274249:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576102.274252:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576102.274258:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576102.274260:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576102.274262:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274268:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576102.274270:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576102.274271:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576102.274273:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=5 00000020:00000001:0.0:1788576102.274275:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576102.274277:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576102.274286:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576102.274288:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576102.274295:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576102.274297:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576102.274303:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576102.274306:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576102.274307:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576102.274311:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576102.274312:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576102.274315:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576102.274317:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576102.274318:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576102.274320:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576102.274323:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30200 with handle 0x1441586093168e32 from hash 00010000:00000001:0.0:1788576102.274325:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576102.274326:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576102.274327:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576102.274328:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576102.274330:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576102.274332:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576102.274333:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576102.274334:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576102.274336:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576102.274338:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576102.274341:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576102.274346:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576102.274347:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576102.274349:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576102.274350:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576102.274359:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576102.274360:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576102.274361:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576102.274364:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e32 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576102.274371:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692200 count: 0 00010000:00000001:0.0:1788576102.274389:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576102.274391:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576102.274394:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576102.274398:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810518528/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576102.274405:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576102.274408:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576102.274409:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576102.274411:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576102.274414:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4ae00. 02000000:00000001:0.0:1788576102.274418:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576102.274420:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576102.274421:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576102.274425:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576102.274426:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576102.274428:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576102.274430:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576102.274433:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576106.113760:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311245087, at 4311245120 00000100:00000010:0.0:1788576106.113778:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303912a0. 00000100:00000040:0.0:1788576111.169862:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311250142, at 4311250176 00000100:00000010:0.0:1788576111.169877:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303913c0. 00000100:00000040:0.0:1788576116.289749:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311255265, at 4311255296 00000100:00000010:0.0:1788576116.289767:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ba0. 00000100:00000040:0.0:1788576121.409831:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311260383, at 4311260416 00000100:00000010:0.0:1788576121.409847:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391c00. 10000000:00000001:0.0:1788576122.689777:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576122.689791:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c372e9a95c0. 00000020:00000010:0.0:1788576122.689795:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c371ed4ae00. 00000020:00000010:0.0:1788576122.689800:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788576122.689812:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788576122.689817:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576122.689822:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788576122.689826:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576122.689830:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788576122.689836:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371985d600. 10000000:00000040:0.0:1788576122.689840:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576122.689843:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576122.689848:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c371ed4be00. 00000040:00000010:0.0:1788576122.689855:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c371ed4ba00. 00000020:00000010:0.0:1788576122.689858:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ed800. 00000020:00000010:0.0:1788576122.689861:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4b600. 80000000:00000010:0.0:1788576122.689864:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626f60. 00040000:00000010:0.0:1788576122.689867:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788576122.689869:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371985d600. 00000020:00000010:0.0:1788576122.689872:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c371ed4ae00. 10000000:00000010:0.0:1788576122.689874:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c372e9a95c0. 10000000:01000000:0.0:1788576122.689877:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576122.689880:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576122.689882:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576122.689884:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576122.689886:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576122.689888:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576122.689890:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576122.689894:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576122.689895:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576122.689897:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576122.689900:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576122.689902:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576122.689904:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1788576124.417891:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576124.417898:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576124.417901:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576124.417903:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576124.417904:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576124.417905:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576124.417907:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576124.417909:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576124.417911:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576124.417914:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576124.417916:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576124.417928:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576124.417933:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576124.417938:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576124.417941:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576124.417947:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576124.417983:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985d600. 00000100:00000001:0.0:1788576124.417991:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576124.417995:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576124.417997:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576124.418007:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788576124.418011:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576124.418014:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30200. 00000020:00000001:0.0:1788576124.418019:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576124.418021:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30200 with handle 0x1441586093168e39 to hash 00000020:00000001:0.0:1788576124.418024:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576124.418025:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576124.418028:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576124.418031:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576124.418035:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576124.418039:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576124.418048:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576124.418077:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576124.418083:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576124.418085:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576124.418101:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c619c0. 00000100:00000001:0.0:1788576124.418104:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611200 : -109714523940416 : ffff9c3718c619c0) 00000100:00000001:0.0:1788576124.418107:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576124.418108:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531980544 : -109714177571072 : ffff9c372d6b4700) 00000100:00000001:0.0:1788576124.418113:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576124.418117:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576124.418119:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576124.418121:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576124.418130:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576124.418138:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576124.418142:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576124.418155:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576124.418159:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576124.418166:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576124.418168:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576124.418170:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576124.418173:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c619c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788576124.418175:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576124.418177:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576124.418181:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576124.418183:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576124.418185:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576124.418195:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576124.418203:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b4700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810521856:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576124.418209:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576124.418211:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576124.418214:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576124.418215:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576124.418218:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576124.418225:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576124.418227:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c619c0. 00000100:00000001:0.0:1788576124.418232:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576124.418234:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576124.418237:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576124.418239:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576124.418241:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576124.418243:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=4 00000020:00000001:0.0:1788576124.418245:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576124.418248:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00010000:0.0:1788576124.418252:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418258:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576124.418263:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418269:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576124.418271:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418282:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576124.418283:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576124.418285:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418288:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576124.418289:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576124.418291:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576124.418293:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576124.418296:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576124.418298:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576124.418299:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418303:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576124.418304:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576124.418305:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576124.418306:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=5 00000020:00000001:0.0:1788576124.418307:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576124.418308:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576124.418310:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576124.418311:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576124.418315:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576124.418316:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576124.418319:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576124.418323:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576124.418324:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576124.418326:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576124.418328:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576124.418330:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576124.418331:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576124.418332:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576124.418333:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576124.418335:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30200 with handle 0x1441586093168e39 from hash 00010000:00000001:0.0:1788576124.418336:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576124.418337:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576124.418337:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576124.418338:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576124.418339:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576124.418340:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576124.418341:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576124.418342:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576124.418343:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576124.418344:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576124.418346:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576124.418349:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576124.418349:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576124.418350:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576124.418351:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576124.418352:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576124.418353:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576124.418353:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576124.418355:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e39 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576124.418359:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788576124.418365:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576124.418366:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576124.418368:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576124.418374:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b4700 x1875440810521856/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576124.418379:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576124.418381:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576124.418382:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576124.418385:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576124.418388:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985d600. 02000000:00000001:0.0:1788576124.418392:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576124.418394:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576124.418395:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b4700. 00000100:00000001:0.0:1788576124.418398:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576124.418399:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576124.418401:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576124.418403:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576124.418406:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576128.257781:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311267232, at 4311267264 00000100:00000010:0.0:1788576128.257796:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c37303911e0. 00000100:00000040:0.0:1788576133.697766:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311272671, at 4311272704 00000100:00000010:0.0:1788576133.697781:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391600. 00000800:00000200:0.0:1788576138.498304:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576138.498320:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576138.498335:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576138.498341:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576138.498344:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576138.820502:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311277792, at 4311277826 00000100:00000010:0.0:1788576138.820508:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391a80. 00000100:00080000:0.0:1788576140.098775:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576140.098788:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576140.098791:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576140.098796:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576140.098801:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576140.098824:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16606 00000400:00000200:0.0:1788576140.098835:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576140.098852:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576140.098856:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576140.098860:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.098864:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.098867:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.098870:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576140.098875:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576140.098878:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788576140.098882:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576140.098887:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788576140.098890:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788576140.098892:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788576140.098895:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.098898:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.098901:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.098904:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576140.098908:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788576140.098912:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576140.098915:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576140.098919:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576140.098923:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576140.098927:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.098930:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576140.098933:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576140.098941:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576140.098945:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788576140.098950:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576140.098952:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.098956:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576140.098959:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576140.098964:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576140.099092:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788576140.099108:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.099120:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810523904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576140.099131:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576140.099136:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.099142:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810523904/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576140.099163:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576140.099167:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576140.099170:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576140.099172:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576140.099174:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576140.099176:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576140.099179:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16606 00000400:00000200:0.0:1788576140.099184:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576140.099186:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576140.099189:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576140.099192:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099194:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.099196:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099198:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576140.099201:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576140.099202:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788576140.099203:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576140.099206:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788576140.099217:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788576140.099220:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788576140.099223:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099225:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.099232:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099235:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576140.099237:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788576140.099241:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099243:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576140.099245:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576140.099252:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576140.099255:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099258:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576140.099264:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576140.099270:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099276:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576140.099281:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788576140.099287:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576140.099292:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576140.099295:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099299:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576140.099302:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576140.099306:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576140.099313:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788576140.099319:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.099329:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810524032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576140.099337:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576140.099340:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.099348:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810524032/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576140.099361:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576140.099365:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576140.099371:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576140.099386:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576140.099395:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576140.099400:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576140.099404:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16606 00000400:00000200:0.0:1788576140.099413:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576140.099417:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576140.099420:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576140.099426:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099431:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.099435:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099439:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576140.099445:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576140.099447:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788576140.099451:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576140.099455:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788576140.099460:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788576140.099475:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788576140.099739:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099744:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.099749:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099754:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576140.099758:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788576140.099765:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099772:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576140.099777:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576140.099782:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576140.099786:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099790:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576140.099795:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576140.099800:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099860:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576140.099872:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788576140.099877:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576140.099882:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576140.099884:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.099889:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576140.099891:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576140.099893:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576140.099897:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788576140.099901:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.099907:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810524160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576140.099914:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576140.099917:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.099921:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810524160/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576140.099929:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576140.099932:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576140.099935:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576140.099937:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576140.099938:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576140.099941:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576140.099943:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16606 00000400:00000200:0.0:1788576140.099948:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576140.099951:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576140.099953:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576140.099956:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099958:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.099960:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.099962:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576140.099964:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576140.100040:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788576140.100043:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576140.100046:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788576140.100048:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788576140.100051:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788576140.100054:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100057:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.100059:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100061:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576140.100063:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788576140.100066:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100069:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576140.100071:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576140.100074:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576140.100076:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100079:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576140.100083:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576140.100086:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100090:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576140.100096:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788576140.100101:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788576140.100106:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576140.100110:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100113:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576140.100115:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576140.100117:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576140.100121:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788576140.100124:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.100130:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810524288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576140.100137:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576140.100140:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.100150:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810524288/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576140.100159:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576140.100162:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576140.100166:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576140.100167:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576140.100169:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576140.100172:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576140.100175:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16606 00000400:00000200:0.0:1788576140.100180:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576140.100183:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576140.100186:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576140.100190:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100192:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.100195:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100197:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576140.100200:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576140.100201:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788576140.100203:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576140.100205:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788576140.100270:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788576140.100272:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788576140.100276:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100278:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576140.100280:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100282:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576140.100284:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788576140.100287:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576140.100290:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576140.100297:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576140.100300:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576140.100302:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100304:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576140.100306:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576140.100309:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100311:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576140.100315:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788576140.100318:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788576140.100320:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576140.100322:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.100324:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576140.100325:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576140.100327:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576140.100331:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788576140.100336:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.100342:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810524416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576140.100354:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576140.100358:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.100367:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810524416/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576140.100437:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576140.100443:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16616) 00000100:00000040:0.0:1788576143.937788:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311282912, at 4311282944 00000100:00000010:0.0:1788576143.937805:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391540. 00000100:00080000:0.0:1788576145.217875:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576145.217889:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576145.217892:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576145.217897:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576145.217902:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576145.217912:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16611 00000400:00000200:0.0:1788576145.217947:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576145.217955:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576145.217959:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576145.217962:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.217966:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.217969:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.217971:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576145.217976:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576145.217978:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788576145.217981:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576145.217984:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788576145.217987:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788576145.217989:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788576145.217992:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.217994:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.217997:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.217999:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576145.218002:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788576145.218006:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218008:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576145.218308:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576145.218313:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576145.218317:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218320:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576145.218323:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576145.218333:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788576145.218337:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788576145.218342:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576145.218343:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218347:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576145.218351:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576145.218356:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576145.218370:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d800. 00000100:00000001:0.0:1788576145.218378:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.218391:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810524544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576145.218403:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576145.218409:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.218415:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810524544/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576145.218447:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576145.218451:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576145.218455:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576145.218456:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576145.218459:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576145.218462:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576145.218465:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16611 00000400:00000200:0.0:1788576145.218469:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576145.218472:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576145.218475:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576145.218478:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218480:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218482:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218484:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576145.218487:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576145.218488:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788576145.218490:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576145.218493:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788576145.218495:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788576145.218497:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788576145.218500:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218513:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218515:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218517:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576145.218520:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788576145.218523:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218525:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576145.218527:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576145.218533:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576145.218536:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218538:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576145.218542:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576145.218546:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218548:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576145.218551:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788576145.218554:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576145.218557:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576145.218558:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218560:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576145.218561:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576145.218563:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576145.218570:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788576145.218574:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.218579:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810524672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576145.218586:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576145.218589:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.218593:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810524672/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576145.218600:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576145.218603:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576145.218606:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576145.218613:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576145.218615:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576145.218618:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576145.218621:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16611 00000400:00000200:0.0:1788576145.218625:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576145.218628:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576145.218631:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576145.218633:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218676:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218679:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218681:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576145.218684:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576145.218685:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788576145.218687:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576145.218689:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788576145.218692:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788576145.218693:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788576145.218697:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218699:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218700:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218702:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576145.218704:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788576145.218707:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218710:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576145.218712:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576145.218715:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576145.218717:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218719:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576145.218722:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576145.218724:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218731:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576145.218734:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788576145.218737:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576145.218739:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576145.218740:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218742:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576145.218743:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576145.218745:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576145.218749:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788576145.218753:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.218758:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810524800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576145.218765:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576145.218767:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.218771:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810524800/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576145.218779:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576145.218782:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576145.218786:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576145.218787:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576145.218789:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576145.218791:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576145.218794:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16611 00000400:00000200:0.0:1788576145.218798:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576145.218801:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576145.218803:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576145.218806:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218808:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218809:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218812:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576145.218814:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576145.218820:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788576145.218821:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576145.218824:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788576145.218825:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788576145.218828:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788576145.218831:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218833:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218835:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218836:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576145.218839:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788576145.218841:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218844:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576145.218845:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576145.218848:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576145.218850:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218851:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576145.218854:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576145.218855:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218857:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576145.218861:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788576145.218863:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788576145.218865:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576145.218866:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218868:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576145.218869:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576145.218871:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576145.218874:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788576145.218877:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.218881:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810524928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576145.218887:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576145.218889:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.218897:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810524928/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576145.218904:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576145.218907:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576145.218910:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576145.218911:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576145.218913:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576145.218916:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576145.218918:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16611 00000400:00000200:0.0:1788576145.218921:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576145.218924:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576145.218926:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576145.218929:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218931:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218932:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218935:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576145.218937:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576145.218938:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788576145.218939:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576145.218941:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788576145.218943:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788576145.218945:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788576145.218948:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218950:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576145.218951:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218953:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576145.218955:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788576145.218957:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576145.218959:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576145.218964:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576145.218966:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576145.218968:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218970:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576145.218972:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576145.218974:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218976:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576145.218980:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788576145.218982:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788576145.218984:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576145.218985:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576145.218987:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576145.218988:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576145.218990:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576145.218993:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f800. 00000100:00000001:0.0:1788576145.218997:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.219000:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810525056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576145.219007:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576145.219009:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.219023:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810525056/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576145.219031:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576145.219036:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16621) 10000000:00000001:0.0:1788576146.373452:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576146.373460:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576146.373464:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576146.373467:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576146.373468:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576146.373469:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576146.373472:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576146.373474:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576146.373476:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576146.373479:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576146.373495:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576146.373505:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788576146.373510:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576146.373515:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576146.373519:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576146.373524:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576146.373532:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371985d600. 00000100:00000001:0.0:1788576146.373539:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576146.373544:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576146.373547:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576146.373554:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c3720692c00. 00010000:00000001:0.0:1788576146.373558:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576146.373561:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db31a00. 00000020:00000001:0.0:1788576146.373565:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576146.373568:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db31a00 with handle 0x1441586093168e40 to hash 00000020:00000001:0.0:1788576146.373570:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576146.373572:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788576146.373574:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788576146.373577:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576146.373581:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576146.373585:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576146.373595:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576146.373607:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576146.373613:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576146.373615:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576146.373621:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c619c0. 00000100:00000001:0.0:1788576146.373623:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185611200 : -109714523940416 : ffff9c3718c619c0) 00000100:00000001:0.0:1788576146.373626:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576146.373633:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531989504 : -109714177562112 : ffff9c372d6b6a00) 00000100:00000001:0.0:1788576146.373652:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576146.373657:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576146.373659:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576146.373661:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576146.373670:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576146.373679:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576146.373684:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576146.373690:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576146.373694:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576146.373700:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576146.373703:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576146.373705:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576146.373707:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c619c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788576146.373709:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576146.373712:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576146.373715:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576146.373717:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576146.373719:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576146.373723:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576146.373730:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810525184:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576146.373736:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576146.373738:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576146.373740:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576146.373742:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576146.373746:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576146.373762:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576146.373864:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c619c0. 00000100:00000001:0.0:1788576146.373870:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576146.373871:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576146.373874:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576146.373877:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576146.373879:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576146.373881:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=4 00000020:00000001:0.0:1788576146.373884:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788576146.373887:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00010000:0.0:1788576146.373890:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.373897:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576146.373901:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.373907:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576146.373910:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.373916:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576146.373918:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576146.373920:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.373925:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576146.373926:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576146.373929:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576146.373932:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576146.373947:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db31a00 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576146.373949:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576146.374135:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.374143:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576146.374144:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576146.374145:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576146.374147:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db31a00 refcount=5 00000020:00000001:0.0:1788576146.374149:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788576146.374151:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536687616 : -109714172864000 : ffff9c372db31a00) 00010000:00000001:0.0:1788576146.374154:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576146.374157:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576146.374163:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576146.374165:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576146.374171:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576146.374173:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576146.374175:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576146.374178:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576146.374179:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576146.374182:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576146.374183:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576146.374185:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576146.374187:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576146.374189:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db31a00 with handle 0x1441586093168e40 from hash 00010000:00000001:0.0:1788576146.374191:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576146.374192:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576146.374193:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576146.374199:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576146.374201:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576146.374202:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576146.374204:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576146.374205:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576146.374207:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576146.374209:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576146.374212:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576146.374218:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576146.374219:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576146.374220:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576146.374221:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576146.374223:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576146.374224:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576146.374225:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576146.374228:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db31a00/0x1441586093168e40 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576146.374234:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c3720692c00 count: 0 00010000:00000001:0.0:1788576146.374242:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576146.374244:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576146.374246:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576146.374251:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b6a00 x1875440810525184/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576146.374257:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576146.374260:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576146.374261:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576146.374263:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576146.374266:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371985d600. 02000000:00000001:0.0:1788576146.374270:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576146.374272:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576146.374273:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b6a00. 00000100:00000001:0.0:1788576146.374277:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576146.374278:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576146.374283:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576146.374285:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576146.374288:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000040:0.0:1788576150.209841:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311289188, at 4311289216 00000100:00000010:0.0:1788576150.209857:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391780. 00000100:00080000:0.0:1788576151.874840:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576151.874850:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576151.874853:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576151.874857:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576151.874861:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576151.874868:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16618 00000400:00000200:0.0:1788576151.874877:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576151.874883:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576151.874887:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576151.874890:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.874893:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.874895:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.874897:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576151.874902:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576151.874903:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788576151.874906:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576151.874910:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788576151.874912:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788576151.874915:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788576151.874918:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.874920:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.874921:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.874923:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576151.874926:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788576151.874930:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576151.874949:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576151.874952:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576151.874955:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576151.874959:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875018:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576151.875021:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576151.875038:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6a00. 00000020:00000040:0.0:1788576151.875042:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788576151.875046:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576151.875048:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875050:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576151.875053:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576151.875056:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576151.875064:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788576151.875069:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.875078:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6a00 x1875440810525952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576151.875087:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576151.875149:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.875155:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6f4c980] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6a00 x1875440810525952/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576151.875178:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576151.875181:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576151.875184:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576151.875186:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576151.875187:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576151.875190:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576151.875193:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16618 00000400:00000200:0.0:1788576151.875197:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576151.875202:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576151.875204:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576151.875208:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875219:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875225:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875228:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576151.875232:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576151.875235:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788576151.875238:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576151.875243:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788576151.875247:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788576151.875249:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788576151.875252:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875256:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875261:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875264:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576151.875270:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788576151.875276:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875281:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576151.875284:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576151.875292:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576151.875296:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875298:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576151.875304:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576151.875308:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875310:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576151.875314:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5f80. 00000020:00000040:0.0:1788576151.875319:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576151.875323:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576151.875325:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875329:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576151.875331:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576151.875333:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576151.875341:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e800. 00000100:00000001:0.0:1788576151.875348:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.875363:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5f80 x1875440810526080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576151.875377:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576151.875381:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.875389:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000759280f1] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5f80 x1875440810526080/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576151.875409:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576151.875411:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576151.875414:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576151.875416:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576151.875418:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576151.875420:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576151.875423:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16618 00000400:00000200:0.0:1788576151.875427:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576151.875430:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576151.875432:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576151.875435:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875437:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875438:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875440:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576151.875443:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576151.875444:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788576151.875445:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576151.875447:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788576151.875449:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788576151.875451:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788576151.875454:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875455:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875457:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875468:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576151.875472:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788576151.875480:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875487:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576151.875491:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576151.875495:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576151.875501:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875504:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576151.875508:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576151.875512:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875516:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576151.875522:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b7480. 00000020:00000040:0.0:1788576151.875534:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576151.875537:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576151.875538:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.875540:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576151.875541:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576151.875543:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576151.875547:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e000. 00000100:00000001:0.0:1788576151.875551:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.875556:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b7480 x1875440810526208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576151.875562:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576151.875564:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.875568:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000562d8f21] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b7480 x1875440810526208/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576151.875580:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576151.875584:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576151.875592:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576151.875597:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576151.875600:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576151.875611:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576151.875617:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16618 00000400:00000200:0.0:1788576151.875626:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576151.875655:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576151.875659:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576151.875663:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875668:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875673:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875677:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576151.875681:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576151.875684:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788576151.875687:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576151.875691:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788576151.875696:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788576151.875699:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788576151.875957:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.875960:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.875962:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876038:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576151.876042:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788576151.876046:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876048:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576151.876050:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576151.876053:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576151.876055:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876057:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576151.876060:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576151.876062:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876063:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576151.876067:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6680. 00000020:00000040:0.0:1788576151.876075:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788576151.876077:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576151.876079:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876080:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576151.876081:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576151.876083:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576151.876087:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20e400. 00000100:00000001:0.0:1788576151.876158:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.876165:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6680 x1875440810526336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576151.876173:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576151.876175:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.876179:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d09fe837] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6680 x1875440810526336/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576151.876187:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576151.876190:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576151.876193:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576151.876194:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576151.876196:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576151.876198:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576151.876201:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16618 00000400:00000200:0.0:1788576151.876205:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576151.876208:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576151.876210:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576151.876212:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876214:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.876216:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876218:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576151.876221:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576151.876222:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788576151.876231:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576151.876233:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788576151.876235:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788576151.876236:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788576151.876240:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876242:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576151.876243:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876245:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576151.876247:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788576151.876250:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576151.876252:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576151.876254:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576151.876257:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576151.876259:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876261:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576151.876263:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576151.876265:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876267:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576151.876270:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6d80. 00000020:00000040:0.0:1788576151.876272:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788576151.876274:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576151.876276:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.876277:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576151.876279:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576151.876280:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576151.876284:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f400. 00000100:00000001:0.0:1788576151.876287:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.876292:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6d80 x1875440810526464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576151.876299:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576151.876301:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.876304:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020d39dd3] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6d80 x1875440810526464/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576151.876317:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576151.876322:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16628) 00000100:00000040:0.0:1788576155.716157:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311294688, at 4311294722 00000100:00000010:0.0:1788576155.716172:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb300. 00000100:00000040:0.0:1788576160.833816:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311299808, at 4311299840 00000100:00000010:0.0:1788576160.833830:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb780. 00000100:00000040:0.0:1788576165.954814:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311304927, at 4311304961 00000100:00000010:0.0:1788576165.954822:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c371ecbb2a0. 00000100:00000040:0.0:1788576172.161762:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311311137, at 4311311168 00000100:00000010:0.0:1788576172.161780:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391720. 00000100:00000040:0.0:1788576177.217786:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311316191, at 4311316224 00000100:00000010:0.0:1788576177.217800:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f60. 00000800:00000200:0.0:1788576177.410131:0:2395:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576177.410138:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576177.410148:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576177.410154:0:2395:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576177.410156:0:2395:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576182.337776:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311321310, at 4311321344 00000100:00000010:0.0:1788576182.337792:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391ae0. 00000100:00080000:0.0:1788576183.618221:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576183.618237:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576183.618240:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576183.618244:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576183.618250:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576183.618259:0:232849:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16650 00000400:00000200:0.0:1788576183.618271:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576183.618279:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576183.618283:0:232849:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576183.618287:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.618311:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.618315:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.618317:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576183.618322:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576183.618325:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788576183.618328:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576183.618332:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788576183.618335:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788576183.618337:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788576183.618340:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.618342:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.618344:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.618346:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576183.618349:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788576183.618353:0:232849:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576183.618356:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576183.618360:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576183.618365:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576183.618368:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.618371:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576183.618373:0:232849:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576183.618380:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b6300. 00000020:00000040:0.0:1788576183.618385:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788576183.618390:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576183.618391:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.618395:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576183.618398:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576183.618403:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576183.618411:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c800. 00000100:00000001:0.0:1788576183.618420:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.618431:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b6300 x1875440810530560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576183.618454:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576183.618460:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.618465:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f32831c] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b6300 x1875440810530560/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576183.621695:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576183.621703:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576183.621707:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576183.621710:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576183.621713:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576183.621717:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576183.621725:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16650 00000400:00000200:0.0:1788576183.621737:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576183.621745:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576183.621749:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576183.621752:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.621756:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.621759:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.621761:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576183.621765:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576183.621768:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788576183.621771:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576183.621977:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788576183.621982:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788576183.621984:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788576183.621989:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622152:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.622155:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622158:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576183.622186:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788576183.622191:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622193:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576183.622197:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576183.622207:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576183.622211:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.622213:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576183.622219:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576183.622222:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.622224:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576183.622232:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4a80. 00000020:00000040:0.0:1788576183.622237:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576183.622242:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576183.622244:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.622247:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576183.622250:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576183.622255:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576183.622265:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20ec00. 00000100:00000001:0.0:1788576183.622274:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.622286:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4a80 x1875440810530688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576183.622297:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576183.622303:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.622308:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002be4e5ae] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4a80 x1875440810530688/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576183.622318:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576183.622321:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576183.622325:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576183.622326:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576183.622328:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576183.622331:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576183.622341:0:232849:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16650 00000400:00000200:0.0:1788576183.622347:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576183.622351:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576183.622353:0:232849:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576183.622357:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622359:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.622362:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622364:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576183.622367:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576183.622368:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788576183.622370:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576183.622372:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788576183.622375:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788576183.622377:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788576183.622380:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622382:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.622384:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622386:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576183.622388:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788576183.622391:0:232849:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576183.622393:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576183.622395:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576183.622398:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576183.623160:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623168:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576183.623172:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576183.623176:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623178:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576183.623186:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5880. 00000020:00000040:0.0:1788576183.623190:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576183.623195:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576183.623205:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623208:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576183.623211:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576183.623214:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576183.623221:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20dc00. 00000100:00000001:0.0:1788576183.623227:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.623234:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5880 x1875440810530816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576183.623244:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576183.623248:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.623252:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000512d8ae2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5880 x1875440810530816/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576183.623261:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576183.623264:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576183.623268:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576183.623269:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576183.623271:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576183.623274:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576183.623277:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16650 00000400:00000200:0.0:1788576183.623283:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576183.623288:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576183.623290:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576183.623293:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.623296:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.623298:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.623300:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576183.623302:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576183.623304:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788576183.623306:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576183.623309:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788576183.623311:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788576183.623320:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788576183.623324:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.623326:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.623328:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.623330:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576183.623332:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788576183.623334:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576183.623336:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576183.623339:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576183.623342:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576183.623345:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623347:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576183.623349:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576183.623352:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623353:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576183.623358:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4000. 00000020:00000040:0.0:1788576183.623360:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788576183.623363:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576183.623364:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.623366:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576183.623367:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576183.623369:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576183.623374:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20d000. 00000100:00000001:0.0:1788576183.623378:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.623382:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4000 x1875440810530944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576183.623389:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576183.623391:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.623395:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000340163ac] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4000 x1875440810530944/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576183.625924:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576183.625954:0:232849:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576183.625959:0:232849:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576183.625962:0:232849:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576183.625965:0:232849:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576183.625969:0:232849:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576183.625976:0:232849:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16650 00000400:00000200:0.0:1788576183.625987:0:232849:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576183.626677:0:232849:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576183.626682:0:232849:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576183.626686:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.626690:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.626692:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.626695:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576183.626698:0:232849:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576183.626701:0:232849:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788576183.626703:0:232849:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576183.626707:0:232849:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788576183.626710:0:232849:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788576183.626712:0:232849:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788576183.626715:0:232849:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.626717:0:232849:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576183.626719:0:232849:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576183.626720:0:232849:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576183.626724:0:232849:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788576183.626727:0:232849:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576183.626730:0:232849:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576183.626733:0:232849:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576183.626741:0:232849:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576183.626744:0:232849:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.626746:0:232849:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576183.626761:0:232849:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576183.626764:0:232849:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.626766:0:232849:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576183.626772:0:232849:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4700. 00000020:00000040:0.0:1788576183.627698:0:232849:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788576183.627707:0:232849:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576183.627709:0:232849:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.627713:0:232849:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576183.627717:0:232849:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576183.627722:0:232849:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576183.627732:0:232849:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788576183.627741:0:232849:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.627750:0:232849:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4700 x1875440810531072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576183.627762:0:232849:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576183.627767:0:232849:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.627773:0:232849:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9deee2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4700 x1875440810531072/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576183.628138:0:232849:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.628145:0:232849:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16660) 00000020:00000001:0.0:1788576185.538100:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576185.538105:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576185.538109:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576187.457773:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311326436, at 4311326464 00000100:00000010:0.0:1788576187.457793:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391f00. 10000000:00000001:0.0:1788576188.744148:0:237086:0:(mgc_request.c:1669:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1788576188.744171:0:237086:0:(mgc_request.c:1683:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff9c371f00f3c0. 00000020:00000010:0.0:1788576188.744182:0:237086:0:(lu_object.c:1758:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9c3707f38200. 00000020:00000010:0.0:1788576188.744189:0:237086:0:(lu_object.c:568:lu_global_key_init()) kmalloced '(value)': 512 at ffff9c3707f38e00. 00000040:00000010:0.0:1788576188.744196:0:237086:0:(llog_obd.c:218:llog_key_init()) kmalloced '(value)': 480 at ffff9c3707f39400. 00000020:00000010:0.0:1788576188.744203:0:237086:0:(dt_object.c:33:dt_global_key_init()) kmalloced '(value)': 1264 at ffff9c37109ee000. 00000020:00000010:0.0:1788576188.744211:0:237086:0:(update_records.c:1179:update_key_init()) kmalloced '(value)': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788576188.744217:0:237086:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff9c3711626240. 00040000:00000010:0.0:1788576188.744240:0:237086:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788576188.744246:0:237086:0:(mgs_handler.c:1787:mgs_key_init()) kmalloced '(value)': 392 at ffff9c371ed4b600. 10000000:00000040:0.0:1788576188.744252:0:237086:0:(lustre_log.h:374:llog_ctxt_get()) GETting ctxt ffff9c3706a69840 : new refcount 2 10000000:00000001:0.0:1788576188.744255:0:237086:0:(mgc_request.c:1715:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000020:00000010:0.0:1788576188.744260:0:237086:0:(lu_object.c:568:lu_global_key_fini()) kfreed 'info': 512 at ffff9c3707f38e00. 00000040:00000010:0.0:1788576188.744268:0:237086:0:(llog_obd.c:218:llog_key_fini()) kfreed 'info': 480 at ffff9c3707f39400. 00000020:00000010:0.0:1788576188.744271:0:237086:0:(dt_object.c:34:dt_global_key_fini()) kfreed 'info': 1264 at ffff9c37109ee000. 00000020:00000010:0.0:1788576188.744275:0:237086:0:(update_records.c:1175:update_key_fini()) kfreed 'info': 432 at ffff9c371ed4ae00. 80000000:00000010:0.0:1788576188.744277:0:237086:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff9c3711626240. 00040000:00000010:0.0:1788576188.744280:0:237086:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff9c371ed4b000. 20000000:00000010:0.0:1788576188.744283:0:237086:0:(mgs_handler.c:1787:mgs_key_fini()) kfreed 'info': 392 at ffff9c371ed4b600. 00000020:00000010:0.0:1788576188.744285:0:237086:0:(lu_object.c:1686:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9c3707f38200. 10000000:00000010:0.0:1788576188.744287:0:237086:0:(mgc_request.c:1731:mgc_process_cfg_log()) kfreed 'env': 56 at ffff9c371f00f3c0. 10000000:01000000:0.0:1788576188.744289:0:237086:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.202.148@tcp: configuration from log 'lustre-cliir' failed (-108). 10000000:00000400:0.0:1788576188.744292:0:237086:0:(mgc_request.c:1900:mgc_process_log()) MGC192.168.202.148@tcp: IR log lustre-cliir failed, not fatal: rc = -108 10000000:00000001:0.0:1788576188.744295:0:237086:0:(mgc_request.c:719:mgc_requeue_add()) Process entered 10000000:00000040:0.0:1788576188.744296:0:237086:0:(mgc_request.c:723:mgc_requeue_add()) log lustre-cliir: requeue (r=2 sp=0 st=1) 10000000:00000040:0.0:1788576188.744298:0:237086:0:(mgc_request.c:726:mgc_requeue_add()) lockh 0x0 00010000:00000001:0.0:1788576188.744300:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1788576188.744303:0:237086:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576188.744305:0:237086:0:(mgc_request.c:742:mgc_requeue_add()) Process leaving 10000000:00000001:0.0:1788576188.744306:0:237086:0:(mgc_request.c:1907:mgc_process_log()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576188.744308:0:237086:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1788576188.744311:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1788576188.744312:0:237086:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576188.744314:0:237086:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:0.0:1788576188.745072:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576188.745077:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576188.745080:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576188.745095:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745102:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.745105:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745108:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576188.745111:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576188.745121:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.745126:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810531200:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576188.745129:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576188.745135:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576188.745138:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745141:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576188.745146:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576188.745148:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.745155:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576188.745160:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576188.745167:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576188.745177:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531200, portal 25 00000100:00000001:0.0:1788576188.745180:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576188.745182:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923072 : -109714417628544 : ffff9c371f1c4a80) 00000100:00000040:0.0:1788576188.745188:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745195:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576188.745198:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576188.745202:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576188.745205:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810531200, offset 0 00000400:00000200:0.0:1788576188.745211:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745221:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745228:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576188.745235:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576188.745239:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576188.745245:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576188.745248:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745251:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544129:544129:256:4294967295] 192.168.202.148@tcp LPNI seq info [544094:544094:8:4294967295] 00000400:00000200:0.0:1788576188.745256:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745266:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576188.745273:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.745284:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788576188.745293:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576188.745298:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788576188.745312:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.745315:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788576188.745320:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576188.745325:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576188.745333:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576188.745335:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576188.745340:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576188.745344:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745353:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.745356:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745363:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.745365:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576188.745371:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.745376:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576188.745379:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4311331557 00000100:00000010:0.0:1788576188.745383:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626240. 00000100:00000001:0.0:1788576188.745391:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788576188.745399:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4311331557 00000100:00000010:0.0:1788576188.745402:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c37116264e0. 00000100:00000001:0.0:1788576188.745405:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788576188.745408:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576188.745411:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.745413:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576188.745418:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576188/real 1788576188] req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576188.745440:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576188.745442:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576188.745444:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576188.745447:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576188.745449:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576188.745452:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576188.745456:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576188.745464:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745470:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576188.745472:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745475:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.745477:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576188.745481:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745488:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576188.745493:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745500:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576188.745505:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745507:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.745508:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745510:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576188.745512:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576188.745515:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576188.745519:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576188.745521:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576188.745526:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576188.745540:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810531200:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576188.745546:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576188.745549:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745556:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.745557:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745560:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576188.745561:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576188.745563:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.745566:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531328:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576188.745569:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576188.745571:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576188.745573:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745574:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576188.745577:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576188.745579:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.745584:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576188.745587:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576188.745591:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576188.745594:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531328, portal 10 00000100:00000001:0.0:1788576188.745596:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576188.745598:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788576188.745605:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745611:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576188.745613:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576188.745615:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576188.745617:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810531328, offset 0 00000400:00000200:0.0:1788576188.745621:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745631:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745660:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576188.745663:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576188.745666:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576188.745668:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576188.745670:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745673:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544130:544130:256:4294967295] 192.168.202.148@tcp LPNI seq info [544095:544095:8:4294967295] 00000400:00000200:0.0:1788576188.745677:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745682:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576188.745686:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.745690:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788576188.745694:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576188.745697:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788576188.745706:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.745708:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788576188.745710:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576188.745713:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576188.745717:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576188.745718:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576188.745721:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576188.745724:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745731:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.745734:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745739:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.745740:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576188.745744:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576188.745747:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788576188.745750:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576188.745761:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311327776 00000100:00000001:0.0:1788576188.745764:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.745766:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311327776 00000100:00000001:0.0:1788576188.745768:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.745770:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576188.745773:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.745774:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576188.745779:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576188/real 1788576188] req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576188.745788:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576188.745789:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576188.745791:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576188.745793:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576188.745795:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576188.745797:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576188.745800:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576188.745808:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745814:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576188.745816:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745817:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.745819:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576188.745822:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745828:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576188.745831:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745837:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576188.745845:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745846:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.745848:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.745849:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576188.745851:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576188.745864:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576188.745869:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576188.745871:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576188.745874:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576188.745881:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531328:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576188.745885:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576188.745887:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745893:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.745894:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745896:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576188.745897:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576188.745899:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.745902:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531456:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576188.745905:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576188.745907:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576188.745908:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.745910:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576188.745913:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576188.745915:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.745919:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576188.745921:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576188.745925:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576188.745928:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531456, portal 10 00000100:00000001:0.0:1788576188.745940:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576188.745942:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788576188.745949:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.745955:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576188.745957:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576188.745960:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576188.745963:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810531456, offset 0 00000400:00000200:0.0:1788576188.745966:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745970:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745974:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576188.745976:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576188.745979:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576188.745981:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576188.745983:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576188.745986:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544131:544131:256:4294967295] 192.168.202.148@tcp LPNI seq info [544096:544096:8:4294967295] 00000400:00000200:0.0:1788576188.745990:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.745996:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576188.746000:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.746004:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788576188.746008:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576188.746010:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788576188.746028:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.746030:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788576188.746033:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576188.746036:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576188.746040:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576188.746041:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576188.746043:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576188.746047:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746064:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.746068:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746074:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.746076:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576188.746079:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576188.746083:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788576188.746087:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576188.746088:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311327780 00000100:00000001:0.0:1788576188.746091:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.746093:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311327780 00000100:00000001:0.0:1788576188.746095:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.746098:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576188.746100:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.746101:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576188.746105:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576188/real 1788576188] req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576188.746114:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576188.746115:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576188.746116:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576188.746119:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576188.746121:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576188.746123:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576188.746126:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576188.746134:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746141:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576188.746148:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746150:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.746151:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576188.746156:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746162:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576188.746166:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746173:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576188.746175:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746177:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.746178:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746180:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576188.746182:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576188.746185:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576188.746188:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576188.746191:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576188.746194:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576188.746202:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531456:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576188.746206:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746209:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.746212:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4a80 x1875440810531200/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746219:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576188.746221:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576188.746224:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576188.746227:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576188.746230:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576188.746237:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576188.746240:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576188.746243:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788576188.746247:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576188.746249:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576188.746251:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4a80. 00000100:00000001:0.0:1788576188.746255:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576188.746256:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746258:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.746262:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440810531328/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746269:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576188.746271:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576188.746272:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576188.746275:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576188.746276:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576188.746277:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576188.746279:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576188.746280:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788576188.746282:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576188.746284:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576188.746285:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788576188.746287:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576188.746288:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746289:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.746292:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810531456/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.746298:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576188.746299:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576188.746301:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576188.746303:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576188.746304:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576188.746305:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576188.746306:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576188.746307:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d000. 02000000:00000001:0.0:1788576188.746310:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576188.746320:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576188.746322:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788576188.746324:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576188.746326:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746328:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.746336:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576188.746339:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.746342:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576188.746344:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.747331:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576188.747337:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576188.747340:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576188.747347:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.747357:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.747359:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.747363:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576188.747366:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576188.747368:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.747372:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531584:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576188.747375:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576188.747379:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576188.747381:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.747383:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576188.747388:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576188.747390:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.747396:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576188.747400:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576188.747406:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576188.747413:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531584, portal 4 00000100:00000001:0.0:1788576188.747415:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576188.747417:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788576188.747424:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.747430:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576188.747444:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576188.747447:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576188.747450:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810531584, offset 0 00000400:00000200:0.0:1788576188.747456:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576188.747463:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.747470:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576188.747475:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576188.747479:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576188.747482:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576188.747485:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576188.747489:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544132:544132:256:4294967295] 192.168.202.148@tcp LPNI seq info [544097:544097:8:4294967295] 00000400:00000200:0.0:1788576188.747493:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.747502:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576188.747508:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.747513:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788576188.747520:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576188.747524:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788576188.747537:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.747540:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788576188.747545:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576188.747550:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576188.747556:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576188.747559:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576188.747563:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576188.747567:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.747576:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.747580:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.747587:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.747589:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576188.747601:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576188.747605:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788576188.747609:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576188.747612:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311327913 00000100:00000001:0.0:1788576188.747615:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.747617:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311327913 00000100:00000001:0.0:1788576188.747619:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.747622:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576188.747624:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.747627:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576188.748069:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576188/real 1788576188] req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576188.748083:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576188.748085:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576188.748087:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576188.748091:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576188.748094:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576188.748097:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576188.748101:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576188.748109:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748115:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576188.748117:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748120:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.748122:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576188.748193:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748202:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576188.748208:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748224:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576188.748227:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748229:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.748230:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748233:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576188.748237:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576188.748240:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576188.748245:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576188.748248:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576188.748253:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576188.748262:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531584:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576188.748268:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576188.748272:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748278:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.748280:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.748282:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576188.748284:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576188.748287:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.748289:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531712:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576188.748294:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576188.748296:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576188.748298:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576188.748299:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576188.748305:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576188.748307:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576188.748313:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576188.748322:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576188.748327:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576188.748331:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531712, portal 4 00000100:00000001:0.0:1788576188.748333:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576188.748335:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000040:0.0:1788576188.748343:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748349:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576188.748351:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576188.748354:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576188.748356:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810531712, offset 0 00000400:00000200:0.0:1788576188.748361:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576188.748366:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.748371:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576188.748374:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576188.748377:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576188.748380:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576188.748383:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576188.748386:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544133:544133:256:4294967295] 192.168.202.148@tcp LPNI seq info [544098:544098:8:4294967295] 00000400:00000200:0.0:1788576188.748389:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576188.748396:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576188.748401:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.748405:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0200. 00000800:00000200:0.0:1788576188.748410:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576188.748413:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788576188.748425:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576188.748428:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0200. 00000400:00000200:0.0:1788576188.748432:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576188.748437:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576188.748440:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576188.748443:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576188.748474:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576188.748479:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748488:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.748491:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748498:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.748501:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576188.748507:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576188.748511:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788576188.748514:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576188.748517:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311327922 00000100:00000001:0.0:1788576188.748519:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.748521:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311327922 00000100:00000001:0.0:1788576188.748524:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311331557 00000100:00000001:0.0:1788576188.748527:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576188.748529:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.748531:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576188.748534:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576188/real 1788576188] req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576188.748541:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576188.748543:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576188.748545:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576188.748547:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576188.748549:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576188.748551:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576188.748555:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576188.748564:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748582:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576188.748584:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748587:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.748588:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576188.748593:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748600:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576188.748605:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748612:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576188.748614:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748615:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576188.748617:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748619:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576188.748621:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576188.748623:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576188.748627:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576188.748725:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576188.748734:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576188.748743:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810531712:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576188.748748:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748750:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.748755:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810531584/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748761:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576188.748765:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576188.748775:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576188.748779:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576188.748784:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576188.748786:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788576188.748868:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576188.748872:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788576188.748876:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576188.748878:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576188.748881:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788576188.748884:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576188.748886:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748888:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576188.748892:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440810531712/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576188.748898:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576188.748900:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576188.748901:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576188.748904:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576188.748905:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576188.748906:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788576188.748908:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576188.748910:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788576188.748913:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576188.748914:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576188.748915:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788576188.748918:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576188.748919:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748921:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.748933:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576188.748935:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.748938:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576188.748940:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576190.659424:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576190.659431:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576190.659436:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1788576190.722211:0:237086:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1788576190.722220:0:237086:0:(mgc_request.c:108:config_log_get()) log lustre-cliir (ffff9c3709498000) refs 2 10000000:00000001:0.0:1788576190.722241:0:237086:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576190.722244:0:237086:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1788576190.722245:0:237086:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1788576190.722246:0:237086:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1788576190.722249:0:237086:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:00000001:0.0:1788576190.722251:0:237086:0:(mgc_request.c:1783:mgc_process_log()) Process entered 10000000:01000000:0.0:1788576190.722253:0:237086:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9c3711155800 from 37 10000000:00000001:0.0:1788576190.722256:0:237086:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1788576190.722258:0:237086:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00000100:00000010:0.0:1788576190.722266:0:237086:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788576190.722270:0:237086:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 00000100:00000001:0.0:1788576190.722275:0:237086:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576190.722278:0:237086:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576190.722284:0:237086:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576190.722291:0:237086:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff9c371ed4b600. 00000100:00000001:0.0:1788576190.722297:0:237086:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576190.722302:0:237086:0:(ldlm_request.c:1064:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1788576190.722304:0:237086:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1788576190.722312:0:237086:0:(ldlm_resource.c:1510:ldlm_resource_new()) slab-alloced 'res': 256 at ffff9c371fc25300. 00010000:00000001:0.0:1788576190.722316:0:237086:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1788576190.722320:0:237086:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff9c372db30200. 00000020:00000001:0.0:1788576190.722324:0:237086:0:(lustre_handles.c:61:class_handle_hash()) Process entered 00000020:00000040:0.0:1788576190.722326:0:237086:0:(lustre_handles.c:93:class_handle_hash()) added object ffff9c372db30200 with handle 0x1441586093168e4e to hash 00000020:00000001:0.0:1788576190.722328:0:237086:0:(lustre_handles.c:94:class_handle_hash()) Process leaving 00010000:00000001:0.0:1788576190.722330:0:237086:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576190.722333:0:237086:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576190.722336:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576190.722340:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576190.722344:0:237086:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576190.722354:0:237086:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576190.722365:0:237086:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1788576190.722377:0:237086:0:(client.c:3360:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1788576190.722379:0:237086:0:(client.c:1176:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1788576190.722386:0:237086:0:(client.c:1178:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff9c3718c61600. 00000100:00000001:0.0:1788576190.722388:0:237086:0:(client.c:1193:ptlrpc_prep_set()) Process leaving (rc=18446634359185610240 : -109714523941376 : ffff9c3718c61600) 00000100:00000001:0.0:1788576190.722391:0:237086:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.722392:0:237086:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359531984128 : -109714177567488 : ffff9c372d6b5500) 00000100:00000001:0.0:1788576190.722397:0:237086:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576190.722402:0:237086:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.722404:0:237086:0:(client.c:2696:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1788576190.722405:0:237086:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.722418:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.722427:0:237086:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1788576190.722431:0:237086:0:(client.c:1417:ptlrpc_import_delay_req()) @@@ IMP_INVALID req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.722438:0:237086:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576190.722442:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.722449:0:237086:0:(client.c:1865:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576190.722452:0:237086:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576190.722454:0:237086:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.722456:0:237086:0:(client.c:2723:ptlrpc_set_wait()) set ffff9c3718c61600 going to sleep for 0 seconds 00000100:00000001:0.0:1788576190.722459:0:237086:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576190.722461:0:237086:0:(client.c:2113:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1788576190.722465:0:237086:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.722467:0:237086:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.722469:0:237086:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.722473:0:237086:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.722481:0:237086:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c372d6b5500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:874b6a47-e66d-4b10-8108-059b044e1170:237086:1875440810531840:192.168.202.148@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1788576190.722493:0:237086:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.722495:0:237086:0:(client.c:2787:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576190.722498:0:237086:0:(client.c:1244:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1788576190.722499:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.722503:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.722510:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576190.722513:0:237086:0:(client.c:3809:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff9c3718c61600. 00000100:00000001:0.0:1788576190.722518:0:237086:0:(client.c:1281:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1788576190.722520:0:237086:0:(client.c:3379:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00000001:0.0:1788576190.722522:0:237086:0:(ldlm_request.c:661:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1788576190.722525:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576190.722527:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576190.722529:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=4 00000020:00000001:0.0:1788576190.722531:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576190.722533:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00010000:0.0:1788576190.722537:0:237086:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722543:0:237086:0:(ldlm_request.c:691:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00010000:00010000:0.0:1788576190.722559:0:237086:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722565:0:237086:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1788576190.722569:0:237086:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722575:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576190.722577:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1788576190.722579:0:237086:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722595:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576190.722596:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788576190.722599:0:237086:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1788576190.722602:0:237086:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788576190.722608:0:237086:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9c372db30200 already unused, calling callback (ffffffffc1255c20) 10000000:00000001:0.0:1788576190.722611:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576190.722614:0:237086:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722619:0:237086:0:(ldlm_request.c:1770:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1788576190.722621:0:237086:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1788576190.722622:0:237086:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576190.722624:0:237086:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff9c372db30200 refcount=5 00000020:00000001:0.0:1788576190.722626:0:237086:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576190.722628:0:237086:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446634359536681472 : -109714172870144 : ffff9c372db30200) 00010000:00000001:0.0:1788576190.722630:0:237086:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788576190.722633:0:237086:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1788576190.722681:0:237086:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1788576190.722683:0:237086:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788576190.722689:0:237086:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:00000040:0.0:1788576190.722691:0:237086:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1788576190.722693:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576190.722696:0:237086:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1788576190.722697:0:237086:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788576190.722700:0:237086:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788576190.722707:0:237086:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788576190.722709:0:237086:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788576190.722711:0:237086:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788576190.722713:0:237086:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff9c372db30200 with handle 0x1441586093168e4e from hash 00010000:00000001:0.0:1788576190.722716:0:237086:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788576190.722717:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576190.722718:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576190.722719:0:237086:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788576190.722721:0:237086:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788576190.722723:0:237086:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1788576190.722725:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576190.722726:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576190.722728:0:237086:0:(ldlm_request.c:1809:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1788576190.722730:0:237086:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576190.722732:0:237086:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1788576190.722738:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576190.722739:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576190.722740:0:237086:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1788576190.722742:0:237086:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1788576190.722744:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788576190.722744:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576190.722745:0:237086:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788576190.722747:0:237086:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.148@tcp lock: ffff9c372db30200/0x1441586093168e4e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 237086 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1788576190.722754:0:237086:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff9c371fc25300 count: 0 00010000:00000001:0.0:1788576190.722763:0:237086:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788576190.722765:0:237086:0:(ldlm_request.c:1218:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1788576190.722779:0:237086:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.722784:0:237086:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c372d6b5500 x1875440810531840/t0(0) o101->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.722791:0:237086:0:(client.c:2801:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788576190.722793:0:237086:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.722795:0:237086:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576190.722802:0:237086:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.722805:0:237086:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff9c371ed4b600. 02000000:00000001:0.0:1788576190.722809:0:237086:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.722810:0:237086:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.722812:0:237086:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c372d6b5500. 00000100:00000001:0.0:1788576190.722816:0:237086:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.722817:0:237086:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1788576190.722819:0:237086:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 10000000:01000000:0.0:1788576190.722821:0:237086:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1788576190.722824:0:237086:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000001:0.0:1788576190.724236:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576190.724240:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576190.724242:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.724247:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724255:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576190.724256:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.724259:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576190.724262:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576190.724264:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.724267:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810531968:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576190.724271:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576190.724275:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576190.724277:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.724279:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576190.724283:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576190.724284:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.724290:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576190.724293:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576190.724300:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576190.724306:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810531968, portal 25 00000100:00000001:0.0:1788576190.724308:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.724309:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000040:0.0:1788576190.724314:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724327:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576190.724330:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576190.724332:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576190.724335:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810531968, offset 0 00000400:00000200:0.0:1788576190.724340:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576190.724347:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.724352:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576190.724356:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576190.724360:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576190.724363:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576190.724366:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576190.724369:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544134:544134:256:4294967295] 192.168.202.148@tcp LPNI seq info [544099:544099:8:4294967295] 00000400:00000200:0.0:1788576190.724373:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.724382:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576190.724387:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.724391:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576190.724397:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576190.724401:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 51 secs later) 00000800:00000100:0.0:1788576190.724413:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.724416:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576190.724420:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576190.724424:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576190.724429:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576190.724431:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576190.724434:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576190.724438:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724445:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.724448:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724463:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.724465:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576190.724471:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.724475:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576190.724477:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4311333536 00000100:00000001:0.0:1788576190.724484:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4311331557 00000100:00000001:0.0:1788576190.724488:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4311333536 00000100:00000001:0.0:1788576190.724490:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4311331557 00000100:00000001:0.0:1788576190.724493:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576190.724495:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.724497:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576190.724501:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576190/real 1788576190] req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576190.724510:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576190.724511:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576190.724513:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576190.724516:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576190.724517:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576190.724519:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576190.724522:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576190.724530:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724536:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576190.724538:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724540:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.724541:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576190.724545:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724570:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.724575:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724582:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576190.724585:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724586:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.724588:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724590:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576190.724591:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576190.724594:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576190.724597:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576190.724613:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576190.724618:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.724626:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810531968:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576190.724631:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.724656:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724663:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576190.724664:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.724671:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576190.724673:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576190.724675:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.724678:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532096:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576190.724681:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576190.724683:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576190.724684:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.724690:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576190.724693:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576190.724695:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.724698:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576190.724701:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576190.724704:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576190.724707:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532096, portal 10 00000100:00000001:0.0:1788576190.724708:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.724710:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923968 : -109714417627648 : ffff9c371f1c4e00) 00000100:00000040:0.0:1788576190.724715:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724722:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576190.724724:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576190.724726:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576190.724728:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810532096, offset 0 00000400:00000200:0.0:1788576190.724731:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576190.724735:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.724738:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576190.724740:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576190.724742:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576190.724745:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576190.724747:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576190.724749:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544135:544135:256:4294967295] 192.168.202.148@tcp LPNI seq info [544100:544100:8:4294967295] 00000400:00000200:0.0:1788576190.724753:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.724759:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576190.724763:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.724765:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576190.724777:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576190.724780:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 51 secs later) 00000800:00000100:0.0:1788576190.724790:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.724791:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576190.724798:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576190.724801:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576190.724804:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576190.724805:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576190.724807:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576190.724811:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724819:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.724821:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724827:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.724829:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576190.724831:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576190.724844:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788576190.724847:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576190.724849:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311329755 00000100:00000001:0.0:1788576190.724851:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.724853:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311329755 00000100:00000001:0.0:1788576190.724855:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.724857:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576190.724859:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.724861:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576190.724864:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576190/real 1788576190] req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576190.724871:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576190.724872:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576190.724873:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576190.724875:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576190.724877:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576190.724879:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576190.724886:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576190.724893:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724899:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576190.724900:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724902:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.724903:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576190.724906:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724913:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.724917:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724924:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576190.724926:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724927:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.724928:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.724930:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576190.724931:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576190.724933:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576190.724936:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576190.724938:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576190.724941:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.724949:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532096:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576190.724952:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.724955:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.724969:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576190.724971:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.724973:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576190.724975:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576190.724976:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.724979:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532224:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576190.724982:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576190.726089:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576190.726093:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.726096:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576190.726103:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576190.726105:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.726112:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576190.726116:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576190.726126:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576190.726130:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532224, portal 10 00000100:00000001:0.0:1788576190.726132:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.726134:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788576190.726143:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726151:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576190.726154:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576190.726156:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576190.726159:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810532224, offset 0 00000400:00000200:0.0:1788576190.726164:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726168:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726173:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576190.726176:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576190.726179:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576190.726182:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576190.726185:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726196:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544136:544136:256:4294967295] 192.168.202.148@tcp LPNI seq info [544101:544101:8:4294967295] 00000400:00000200:0.0:1788576190.726200:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726206:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576190.726211:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726215:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576190.726220:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576190.726223:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 51 secs later) 00000800:00000100:0.0:1788576190.726233:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726235:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576190.726239:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576190.726242:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576190.726246:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576190.726248:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576190.726251:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576190.726255:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726265:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.726268:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726275:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726276:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576190.726280:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576190.726284:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788576190.726301:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576190.726303:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311329760 00000100:00000001:0.0:1788576190.726306:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726308:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311329760 00000100:00000001:0.0:1788576190.726310:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726312:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576190.726321:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726324:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576190.726328:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576190/real 1788576190] req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576190.726338:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576190.726339:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576190.726341:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576190.726343:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576190.726345:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576190.726347:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576190.726350:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576190.726358:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726366:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576190.726367:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726369:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.726371:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576190.726374:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726381:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.726385:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726391:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576190.726394:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726395:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.726397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726399:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576190.726400:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576190.726403:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576190.726411:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576190.726413:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576190.726416:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.726424:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532224:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576190.726428:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.726431:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726438:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576190.726440:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.726442:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576190.726444:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576190.726446:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.726449:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532352:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576190.726452:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576190.726455:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576190.726456:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.726458:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576190.726461:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576190.726463:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.726466:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576190.726469:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576190.726472:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576190.726475:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532352, portal 4 00000100:00000001:0.0:1788576190.726477:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.726478:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291933824 : -109714417617792 : ffff9c371f1c7480) 00000100:00000040:0.0:1788576190.726483:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726488:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576190.726496:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576190.726498:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576190.726501:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810532352, offset 0 00000400:00000200:0.0:1788576190.726504:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726507:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726510:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576190.726513:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576190.726515:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576190.726517:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576190.726518:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726521:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544137:544137:256:4294967295] 192.168.202.148@tcp LPNI seq info [544102:544102:8:4294967295] 00000400:00000200:0.0:1788576190.726524:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726529:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576190.726534:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726537:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576190.726540:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576190.726543:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 51 secs later) 00000800:00000100:0.0:1788576190.726553:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726555:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576190.726558:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576190.726561:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576190.726564:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576190.726566:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576190.726568:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576190.726572:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726580:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.726583:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726588:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726590:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576190.726598:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576190.726602:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788576190.726605:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576190.726607:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311329892 00000100:00000001:0.0:1788576190.726609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726611:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311329892 00000100:00000001:0.0:1788576190.726613:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726615:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576190.726617:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726619:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576190.726623:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576190/real 1788576190] req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576190.726630:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576190.726631:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576190.726633:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576190.726651:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576190.726653:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576190.726655:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576190.726658:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576190.726666:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726672:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576190.726673:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726675:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.726676:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576190.726679:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726685:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.726688:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726699:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576190.726702:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726703:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.726705:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726706:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576190.726708:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576190.726711:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576190.726714:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576190.726717:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576190.726720:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.726727:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532352:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576190.726731:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576190.726734:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726740:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576190.726741:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.726743:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576190.726745:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576190.726747:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576190.726750:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532480:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576190.726753:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576190.726755:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576190.726756:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576190.726758:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576190.726761:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576190.726763:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576190.726767:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576190.726774:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576190.726777:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576190.726780:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532480, portal 4 00000100:00000001:0.0:1788576190.726781:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576190.726782:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291929344 : -109714417622272 : ffff9c371f1c6300) 00000100:00000040:0.0:1788576190.726788:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726795:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576190.726797:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576190.726799:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576190.726801:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810532480, offset 0 00000400:00000200:0.0:1788576190.726805:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726809:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726813:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576190.726815:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576190.726818:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576190.726820:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576190.726822:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576190.726825:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544138:544138:256:4294967295] 192.168.202.148@tcp LPNI seq info [544103:544103:8:4294967295] 00000400:00000200:0.0:1788576190.726829:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576190.726847:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576190.726852:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726855:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576190.726859:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576190.726861:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 51 secs later) 00000800:00000100:0.0:1788576190.726870:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576190.726872:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576190.726875:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576190.726878:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576190.726880:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576190.726882:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576190.726890:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576190.726893:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726900:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.726903:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726908:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726910:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576190.726913:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576190.726916:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788576190.726919:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576190.726921:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311329902 00000100:00000001:0.0:1788576190.726923:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726925:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311329902 00000100:00000001:0.0:1788576190.726927:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311333536 00000100:00000001:0.0:1788576190.726929:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576190.726931:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.726932:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576190.726936:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576190/real 1788576190] req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576190.726943:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576190.726945:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576190.726946:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576190.726948:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576190.726950:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576190.726952:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576190.726955:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576190.726963:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.726979:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576190.726981:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.726983:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.728843:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576190.728855:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.728865:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576190.728869:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.728878:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576190.728881:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.728883:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576190.728884:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.728886:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576190.728888:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576190.728891:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576190.728895:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576190.728897:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576190.728900:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576190.728909:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532480:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576190.728914:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.728917:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.728920:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440810531968/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.728928:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576190.728930:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576190.728943:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576190.728947:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576190.728950:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.728952:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576190.728954:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.728957:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233cc00. 02000000:00000001:0.0:1788576190.728960:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.728962:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.728964:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788576190.728967:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.728968:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.728970:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.728974:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4e00 x1875440810532096/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.728981:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576190.728982:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576190.729207:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576190.729212:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576190.729213:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.729215:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576190.729216:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.729218:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233ec00. 02000000:00000001:0.0:1788576190.729221:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.729222:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.729223:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4e00. 00000100:00000001:0.0:1788576190.729226:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.729227:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.729229:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.729233:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440810532224/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.729240:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576190.729241:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576190.729242:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576190.729244:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576190.729245:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.729246:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576190.729253:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.729254:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e800. 02000000:00000001:0.0:1788576190.729256:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.729257:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.729259:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788576190.729261:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.729262:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.729263:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.729267:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440810532352/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.729273:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576190.729274:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576190.729276:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576190.729277:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576190.729279:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.729280:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788576190.729281:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.729282:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233dc00. 02000000:00000001:0.0:1788576190.729284:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.729285:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.729286:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788576190.729288:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.729289:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.729290:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576190.729293:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440810532480/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576190.729299:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576190.729300:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576190.729301:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576190.729303:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576190.729304:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576190.729305:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788576190.729306:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576190.729307:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233c400. 02000000:00000001:0.0:1788576190.729309:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576190.729313:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576190.729314:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788576190.729316:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576190.729317:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.729319:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576190.729327:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576190.729329:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576190.729331:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576190.729333:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576194.561782:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311333536, at 4311333568 00000100:00000010:0.0:1788576194.561806:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626240. 00000020:00000001:0.0:1788576195.780563:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576195.780569:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576195.780574:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576196.102690:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576196.102699:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576196.102701:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576196.102714:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.102724:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.102727:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.102731:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576196.102734:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576196.102737:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.102740:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810532608:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576196.102745:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576196.102750:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576196.102752:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.102755:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576196.102765:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576196.102767:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.184424:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576196.184461:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576196.184471:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576196.184478:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532608, portal 25 00000100:00000001:0.0:1788576196.184487:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576196.184488:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286341504 : -109714423210112 : ffff9c371ec71f80) 00000100:00000040:0.0:1788576196.184496:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184503:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576196.184506:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576196.184509:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576196.184512:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810532608, offset 0 00000400:00000200:0.0:1788576196.184518:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576196.184525:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.184532:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576196.184537:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576196.184540:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576196.184543:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576196.184546:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576196.184550:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544139:544139:256:4294967295] 192.168.202.148@tcp LPNI seq info [544104:544104:8:4294967295] 00000400:00000200:0.0:1788576196.184554:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.184563:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576196.184569:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.184575:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576196.184581:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576196.184584:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788576196.184597:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.184600:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576196.184605:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576196.184609:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576196.184614:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576196.184616:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576196.184620:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576196.184624:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184658:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.184662:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184668:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.184669:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576196.184677:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.184681:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576196.184684:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4311338996 00000100:00000010:0.0:1788576196.184689:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626240. 00000100:00000001:0.0:1788576196.184695:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788576196.184700:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4311338996 00000100:00000010:0.0:1788576196.184703:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626b40. 00000100:00000001:0.0:1788576196.184706:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788576196.184720:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576196.184723:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.184727:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576196.184731:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576196/real 1788576196] req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576196.184741:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576196.184742:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576196.184744:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576196.184747:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576196.184748:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576196.184751:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576196.184754:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576196.184761:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184767:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576196.184785:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184788:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.184790:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576196.184795:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184801:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576196.184806:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184813:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576196.184816:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184818:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.184819:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184821:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576196.184823:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576196.184826:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576196.184830:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576196.184837:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576196.184842:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576196.184850:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810532608:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576196.184854:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184857:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.184859:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71f80 x1875440810532608/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184865:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576196.184867:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576196.184869:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576196.184873:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576196.184875:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576196.184881:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576196.184883:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576196.184886:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2c000. 02000000:00000001:0.0:1788576196.184889:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576196.184891:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576196.184893:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71f80. 00000100:00000001:0.0:1788576196.184898:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576196.184899:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184901:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.184908:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576196.184910:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.184912:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576196.184914:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576196.184916:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576196.184920:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184928:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.184930:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.184933:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576196.184936:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576196.184938:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.184942:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532736:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576196.184945:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576196.184949:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576196.184951:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.184953:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576196.184958:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576196.184959:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.184963:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576196.184966:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576196.184969:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576196.184972:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532736, portal 10 00000100:00000001:0.0:1788576196.184974:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576196.184975:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286343296 : -109714423208320 : ffff9c371ec72680) 00000100:00000040:0.0:1788576196.184981:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.184993:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576196.184995:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576196.184998:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576196.185000:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810532736, offset 0 00000400:00000200:0.0:1788576196.185004:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185008:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185012:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576196.185015:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576196.185017:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576196.185020:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576196.185022:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185025:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544140:544140:256:4294967295] 192.168.202.148@tcp LPNI seq info [544105:544105:8:4294967295] 00000400:00000200:0.0:1788576196.185029:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185035:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576196.185039:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185043:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576196.185047:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576196.185050:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788576196.185059:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185061:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576196.185064:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576196.185067:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576196.185070:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576196.185071:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576196.185073:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576196.185077:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185084:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.185087:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185103:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185105:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576196.185109:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576196.185113:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788576196.185116:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576196.185118:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311335214 00000100:00000001:0.0:1788576196.185120:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185122:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311335214 00000100:00000001:0.0:1788576196.185124:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185126:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576196.185128:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185130:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576196.185134:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576196/real 1788576196] req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576196.185140:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576196.185142:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576196.185143:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576196.185145:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576196.185147:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576196.185149:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576196.185151:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576196.185158:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185164:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576196.185165:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185167:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185168:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576196.185171:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185182:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576196.185186:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185192:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576196.185194:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185195:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185196:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185198:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576196.185200:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576196.185202:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576196.185205:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576196.185208:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576196.185211:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576196.185219:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532736:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576196.185223:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576196.185227:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185233:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.185234:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185236:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576196.185238:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576196.185240:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.185243:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532864:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576196.185246:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576196.185247:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576196.185254:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185255:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576196.185258:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576196.185260:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.185264:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576196.185267:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576196.185270:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576196.185273:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532864, portal 10 00000100:00000001:0.0:1788576196.185274:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576196.185276:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286344192 : -109714423207424 : ffff9c371ec72a00) 00000100:00000040:0.0:1788576196.185281:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185286:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576196.185288:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576196.185290:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576196.185292:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810532864, offset 0 00000400:00000200:0.0:1788576196.185296:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185300:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185303:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576196.185306:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576196.185308:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576196.185310:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576196.185312:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185315:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544141:544141:256:4294967295] 192.168.202.148@tcp LPNI seq info [544106:544106:8:4294967295] 00000400:00000200:0.0:1788576196.185318:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185323:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576196.185327:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185330:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576196.185333:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576196.185336:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788576196.185345:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185351:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576196.185354:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576196.185357:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576196.185361:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576196.185362:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576196.185364:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576196.185368:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185375:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.185378:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185384:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185385:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576196.185388:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576196.185391:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788576196.185394:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576196.185395:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311335218 00000100:00000001:0.0:1788576196.185398:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185399:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311335218 00000100:00000001:0.0:1788576196.185401:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185403:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576196.185405:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185407:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576196.185411:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576196/real 1788576196] req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576196.185419:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576196.185420:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576196.185422:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576196.185424:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576196.185426:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576196.185433:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576196.185437:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576196.185444:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185450:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576196.185451:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185453:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185454:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576196.185458:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185464:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576196.185467:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185472:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576196.185474:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185475:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185476:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185477:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576196.185479:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576196.185481:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576196.185484:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576196.185486:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576196.185489:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576196.185496:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532864:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576196.185499:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576196.185507:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185514:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.185515:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185518:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576196.185519:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576196.185521:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.185524:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532992:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576196.185527:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576196.185528:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576196.185530:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185532:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576196.185535:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576196.185536:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.185539:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576196.185542:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576196.185545:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576196.185548:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810532992, portal 4 00000100:00000001:0.0:1788576196.185549:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576196.185550:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286342400 : -109714423209216 : ffff9c371ec72300) 00000100:00000040:0.0:1788576196.185555:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185560:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576196.185562:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576196.185564:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576196.185566:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810532992, offset 0 00000400:00000200:0.0:1788576196.185570:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185574:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185577:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576196.185579:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576196.185582:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576196.185584:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576196.185586:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185595:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544142:544142:256:4294967295] 192.168.202.148@tcp LPNI seq info [544107:544107:8:4294967295] 00000400:00000200:0.0:1788576196.185598:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185604:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576196.185608:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185611:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576196.185614:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576196.185617:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788576196.185627:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185629:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576196.185631:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576196.185652:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576196.185655:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576196.185657:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576196.185660:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576196.185663:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185670:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.185673:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185679:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185680:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576196.185683:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576196.185686:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788576196.185688:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576196.185690:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311335350 00000100:00000001:0.0:1788576196.185692:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185694:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311335350 00000100:00000001:0.0:1788576196.185695:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.185697:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576196.185705:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185707:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576196.185711:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576196/real 1788576196] req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576196.185719:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576196.185720:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576196.185722:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576196.185724:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576196.185725:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576196.185728:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576196.185731:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576196.185739:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185746:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576196.185748:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185750:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185751:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576196.185754:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185761:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576196.185764:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185771:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576196.185784:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185786:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.185787:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.185789:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576196.185792:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576196.185794:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576196.185802:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576196.185804:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576196.185808:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576196.185816:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810532992:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576196.185820:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576196.185823:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185829:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.185830:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185832:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576196.185834:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576196.185836:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.185839:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533120:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576196.185842:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576196.185844:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576196.185845:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576196.185847:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576196.185850:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576196.185852:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576196.185855:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576196.185857:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576196.185860:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576196.185863:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533120, portal 4 00000100:00000001:0.0:1788576196.185865:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576196.185866:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359286340608 : -109714423211008 : ffff9c371ec71c00) 00000100:00000040:0.0:1788576196.185871:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185877:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576196.185885:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576196.185887:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576196.185890:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810533120, offset 0 00000400:00000200:0.0:1788576196.185893:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185897:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185900:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576196.185903:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576196.185905:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576196.185907:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576196.185909:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576196.185912:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544143:544143:256:4294967295] 192.168.202.148@tcp LPNI seq info [544108:544108:8:4294967295] 00000400:00000200:0.0:1788576196.185916:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576196.185921:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576196.185925:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185928:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576196.185932:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576196.185934:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 46 secs later) 00000800:00000100:0.0:1788576196.185950:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576196.185952:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576196.185955:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576196.185959:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576196.185963:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576196.185964:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576196.185966:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576196.185970:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185977:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.185980:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.185986:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.185987:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576196.185996:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576196.186000:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788576196.186003:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576196.186005:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311335359 00000100:00000001:0.0:1788576196.186007:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.186010:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311335359 00000100:00000001:0.0:1788576196.186012:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311338996 00000100:00000001:0.0:1788576196.186014:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576196.186015:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.186017:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576196.186021:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576196/real 1788576196] req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576196.186028:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576196.186030:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576196.186031:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576196.186034:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576196.186035:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576196.186037:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576196.186040:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576196.186048:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186054:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576196.186056:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186057:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.186058:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576196.186062:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186068:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576196.186071:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186083:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576196.186086:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186087:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576196.186089:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186091:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576196.186092:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576196.186094:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576196.186098:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576196.186100:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576196.186104:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576196.186111:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371ec71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533120:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576196.186115:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186117:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.186120:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72680 x1875440810532736/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186126:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576196.186127:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576196.186129:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576196.186132:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576196.186134:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576196.186135:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576196.186137:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576196.186139:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2cc00. 02000000:00000001:0.0:1788576196.186141:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576196.186143:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576196.186144:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72680. 00000100:00000001:0.0:1788576196.186148:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576196.186149:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186155:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.186159:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72a00 x1875440810532864/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186165:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576196.186167:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576196.186168:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576196.186170:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576196.186172:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576196.186173:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576196.186174:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576196.186175:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e000. 02000000:00000001:0.0:1788576196.186177:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576196.186178:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576196.186179:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72a00. 00000100:00000001:0.0:1788576196.186182:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576196.186183:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186184:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.186187:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec72300 x1875440810532992/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186194:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576196.186195:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576196.186196:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576196.186198:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576196.186199:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576196.186200:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788576196.186202:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576196.186203:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2e400. 02000000:00000001:0.0:1788576196.186205:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576196.186206:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576196.186207:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec72300. 00000100:00000001:0.0:1788576196.186209:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576196.186210:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186211:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576196.186214:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371ec71c00 x1875440810533120/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576196.186230:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576196.186231:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576196.186233:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f800. 02000000:00000001:0.0:1788576196.186235:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576196.186236:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576196.186237:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788576196.186239:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576196.186241:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c371ff2fc00. 02000000:00000001:0.0:1788576196.186243:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576196.186244:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576196.186245:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371ec71c00. 00000100:00000001:0.0:1788576196.186248:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576196.186249:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186251:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.186255:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576196.186257:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576196.186259:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576196.186261:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576200.003196:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311338996, at 4311339009 00000100:00000010:0.0:1788576200.003204:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626240. 00000020:00000001:0.0:1788576200.899885:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576200.899893:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576200.899899:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576201.540016:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576201.540023:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576201.540027:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576201.540037:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540047:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.540049:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540053:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576201.540057:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576201.540060:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.540063:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810533248:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576201.540088:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576201.540093:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576201.540096:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540099:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576201.540105:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576201.540107:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.540114:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576201.540119:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576201.540129:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576201.540139:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533248, portal 25 00000100:00000001:0.0:1788576201.540142:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576201.540143:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291931136 : -109714417620480 : ffff9c371f1c6a00) 00000100:00000040:0.0:1788576201.540150:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540158:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576201.540161:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576201.540164:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576201.540168:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810533248, offset 0 00000400:00000200:0.0:1788576201.540175:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576201.540184:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.540192:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576201.540199:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576201.540203:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576201.540208:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576201.540211:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576201.540216:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544144:544144:256:4294967295] 192.168.202.148@tcp LPNI seq info [544109:544109:8:4294967295] 00000400:00000200:0.0:1788576201.540221:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.540232:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576201.540239:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.540244:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576201.540252:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576201.540262:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 40 secs later) 00000800:00000100:0.0:1788576201.540278:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.540281:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576201.540289:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576201.540295:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576201.540303:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576201.540306:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576201.540311:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576201.540315:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540325:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.540329:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540337:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.540339:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576201.540346:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.540351:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576201.540354:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4311344352 00000100:00000010:0.0:1788576201.540359:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626240. 00000100:00000001:0.0:1788576201.540367:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788576201.540375:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4311344352 00000100:00000010:0.0:1788576201.540378:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626f00. 00000100:00000001:0.0:1788576201.540382:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788576201.540385:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576201.540388:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.540391:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576201.540397:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576201/real 1788576201] req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576201.540408:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576201.540409:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576201.540417:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576201.540420:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576201.540422:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576201.540425:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576201.540430:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576201.540439:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540446:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576201.540448:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540451:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.540453:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576201.540458:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540465:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576201.540470:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540477:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576201.540480:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540482:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.540483:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540486:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576201.540488:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576201.540492:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576201.540496:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576201.540519:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576201.540524:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576201.540534:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810533248:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576201.540548:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576201.540553:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540560:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.540562:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540564:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576201.540566:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576201.540569:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.540572:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533376:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576201.540575:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576201.540578:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576201.540580:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540582:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576201.540586:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576201.540588:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.540592:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576201.540595:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576201.540599:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576201.540602:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533376, portal 10 00000100:00000001:0.0:1788576201.540604:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576201.540605:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291929344 : -109714417622272 : ffff9c371f1c6300) 00000100:00000040:0.0:1788576201.540613:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540619:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576201.540621:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576201.540624:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576201.540626:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810533376, offset 0 00000400:00000200:0.0:1788576201.540630:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576201.540690:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.540694:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576201.540697:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576201.540708:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576201.540710:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576201.540713:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576201.540728:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544145:544145:256:4294967295] 192.168.202.148@tcp LPNI seq info [544110:544110:8:4294967295] 00000400:00000200:0.0:1788576201.540733:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.540740:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576201.540745:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.540749:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576201.540752:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576201.540755:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 40 secs later) 00000800:00000100:0.0:1788576201.540766:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.540768:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576201.540772:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576201.540775:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576201.540779:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576201.540781:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576201.540783:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576201.540787:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540795:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.540798:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540804:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.540806:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576201.540809:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576201.540814:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788576201.540817:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576201.540819:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311340571 00000100:00000001:0.0:1788576201.540822:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.540824:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311340571 00000100:00000001:0.0:1788576201.540834:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.540837:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576201.540839:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.540841:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576201.540846:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576201/real 1788576201] req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576201.540854:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576201.540856:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576201.540858:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576201.540860:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576201.540862:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576201.540865:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576201.540868:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576201.540876:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540883:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576201.540884:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540887:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.540888:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576201.540891:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540898:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576201.540902:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540908:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576201.540910:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540912:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.540913:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.540921:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576201.540924:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576201.540926:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576201.540930:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576201.540932:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576201.540937:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576201.540944:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533376:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576201.540949:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576201.540952:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.540959:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.540960:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540962:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576201.540964:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576201.540966:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.540969:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533504:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576201.540973:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576201.540976:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576201.540977:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.540979:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576201.540983:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576201.540985:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.540989:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576201.540991:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576201.540995:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576201.540999:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533504, portal 10 00000100:00000001:0.0:1788576201.541001:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576201.541002:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291933824 : -109714417617792 : ffff9c371f1c7480) 00000100:00000040:0.0:1788576201.541008:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541021:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576201.541024:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576201.541026:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576201.541029:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810533504, offset 0 00000400:00000200:0.0:1788576201.541033:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541037:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541041:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576201.541044:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576201.541046:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576201.541048:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576201.541051:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541054:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544146:544146:256:4294967295] 192.168.202.148@tcp LPNI seq info [544111:544111:8:4294967295] 00000400:00000200:0.0:1788576201.541058:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541064:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576201.541068:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541072:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576201.541076:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576201.541079:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 40 secs later) 00000800:00000100:0.0:1788576201.541089:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541091:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576201.541095:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576201.541099:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576201.541102:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576201.541104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576201.541106:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576201.541110:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.541120:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541137:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541139:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576201.541144:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576201.541148:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788576201.541152:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576201.541154:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311340575 00000100:00000001:0.0:1788576201.541157:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541159:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311340575 00000100:00000001:0.0:1788576201.541161:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541164:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576201.541166:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541168:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576201.541173:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576201/real 1788576201] req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576201.541180:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576201.541182:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576201.541184:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576201.541186:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576201.541188:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576201.541190:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576201.541193:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576201.541201:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541207:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576201.541209:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541210:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541212:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576201.541215:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541227:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576201.541233:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541239:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576201.541242:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541244:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541245:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541247:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576201.541249:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576201.541252:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576201.541256:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576201.541258:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576201.541262:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576201.541270:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533504:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576201.541275:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576201.541279:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541286:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.541288:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.541290:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576201.541291:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576201.541293:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.541296:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533632:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576201.541300:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576201.541302:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576201.541309:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.541311:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576201.541315:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576201.541317:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.541321:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576201.541324:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576201.541328:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576201.541331:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533632, portal 4 00000100:00000001:0.0:1788576201.541333:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576201.541335:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291928448 : -109714417623168 : ffff9c371f1c5f80) 00000100:00000040:0.0:1788576201.541341:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541348:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576201.541351:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576201.541353:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576201.541356:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810533632, offset 0 00000400:00000200:0.0:1788576201.541360:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541364:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541367:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576201.541370:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576201.541373:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576201.541375:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576201.541377:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541380:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544147:544147:256:4294967295] 192.168.202.148@tcp LPNI seq info [544112:544112:8:4294967295] 00000400:00000200:0.0:1788576201.541384:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541390:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576201.541394:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541397:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576201.541401:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576201.541404:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 40 secs later) 00000800:00000100:0.0:1788576201.541415:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541421:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576201.541424:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576201.541428:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576201.541431:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576201.541433:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576201.541436:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576201.541439:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541447:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.541450:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541457:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541458:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576201.541462:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576201.541464:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788576201.541467:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576201.541469:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311340707 00000100:00000001:0.0:1788576201.541472:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541474:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311340707 00000100:00000001:0.0:1788576201.541476:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541478:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576201.541480:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541482:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576201.541486:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576201/real 1788576201] req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576201.541494:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576201.541495:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576201.541497:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576201.541499:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576201.541501:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576201.541511:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576201.541515:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576201.541524:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541530:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576201.541532:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541534:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541536:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576201.541539:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541545:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576201.541550:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541556:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576201.541559:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541560:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541561:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541563:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576201.541566:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576201.541569:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576201.541572:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576201.541574:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576201.541578:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576201.541586:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533632:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576201.541590:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576201.541595:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541608:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.541609:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.541612:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576201.541613:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576201.541616:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.541619:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533760:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576201.541622:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576201.541625:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576201.541627:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576201.541628:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576201.541632:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788576201.541668:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576201.541673:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576201.541677:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576201.541681:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576201.541684:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810533760, portal 4 00000100:00000001:0.0:1788576201.541686:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576201.541687:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291923968 : -109714417627648 : ffff9c371f1c4e00) 00000100:00000040:0.0:1788576201.541694:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541700:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576201.541702:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576201.541704:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576201.541708:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810533760, offset 0 00000400:00000200:0.0:1788576201.541712:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541727:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541731:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576201.541735:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576201.541737:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576201.541739:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576201.541742:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576201.541751:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544148:544148:256:4294967295] 192.168.202.148@tcp LPNI seq info [544113:544113:8:4294967295] 00000400:00000200:0.0:1788576201.541756:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576201.541762:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576201.541768:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541771:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576201.541775:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576201.541778:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 40 secs later) 00000800:00000100:0.0:1788576201.541789:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576201.541790:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576201.541793:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576201.541796:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576201.541800:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576201.541801:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576201.541804:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576201.541808:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541815:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.541818:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541825:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541826:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576201.541829:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576201.541832:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788576201.541836:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576201.541837:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311340717 00000100:00000001:0.0:1788576201.541840:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541842:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311340717 00000100:00000001:0.0:1788576201.541844:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311344352 00000100:00000001:0.0:1788576201.541846:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576201.541855:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.541857:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576201.541861:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576201/real 1788576201] req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576201.541870:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576201.541871:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576201.541873:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576201.541876:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576201.541878:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576201.541880:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576201.541883:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576201.541892:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541899:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576201.541901:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541903:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541904:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576201.541907:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541913:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576201.541916:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541923:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576201.541926:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541927:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576201.541928:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541930:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576201.541932:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576201.541935:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576201.541944:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576201.541946:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576201.541951:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576201.541959:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810533760:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576201.541964:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.541966:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.541970:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6a00 x1875440810533248/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.541977:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576201.541980:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576201.541983:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576201.541986:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576201.541990:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576201.541992:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576201.541994:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576201.541998:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f000. 02000000:00000001:0.0:1788576201.542002:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576201.542004:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576201.542006:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6a00. 00000100:00000001:0.0:1788576201.542010:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576201.542012:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542013:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.542016:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c6300 x1875440810533376/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.542022:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576201.542024:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576201.542025:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576201.542028:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576201.542030:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576201.542031:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576201.542033:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576201.542040:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788576201.542043:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576201.542044:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576201.542045:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c6300. 00000100:00000001:0.0:1788576201.542048:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576201.542049:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542051:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.542055:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7480 x1875440810533504/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.542061:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576201.542063:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576201.542064:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c400. 02000000:00000001:0.0:1788576201.542067:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576201.542068:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576201.542069:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576201.542071:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576201.542072:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788576201.542074:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576201.542075:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576201.542077:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7480. 00000100:00000001:0.0:1788576201.542079:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576201.542080:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542081:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.542085:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5f80 x1875440810533632/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.542091:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576201.542092:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576201.542093:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20f000. 02000000:00000001:0.0:1788576201.542096:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576201.542097:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576201.542098:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788576201.542100:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576201.542102:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d800. 02000000:00000001:0.0:1788576201.542104:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576201.542106:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576201.542107:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5f80. 00000100:00000001:0.0:1788576201.542115:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576201.542117:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542119:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576201.542123:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4e00 x1875440810533760/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576201.542129:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576201.542131:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576201.542132:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20cc00. 02000000:00000001:0.0:1788576201.542134:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576201.542136:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576201.542137:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788576201.542139:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576201.542140:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788576201.542142:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576201.542144:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576201.542145:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4e00. 00000100:00000001:0.0:1788576201.542147:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576201.542149:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542152:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.542162:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576201.542164:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.542167:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576201.542170:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576205.379371:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311344352, at 4311344385 00000100:00000010:0.0:1788576205.379379:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3711626240. 00000020:00000001:0.0:1788576206.080818:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576206.080840:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576206.080847:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788576206.657874:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 874b6a47-e66d-4b10-8108-059b044e1170->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576206.657889:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576206.657892:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576206.657897:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576206.657902:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576206.657911:0:267915:0:(import.c:545:import_select_connection()) MGC192.168.202.148@tcp: connect to NID 192.168.202.148@tcp last attempt 16673 00000400:00000200:0.0:1788576206.657924:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576206.657959:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576206.657963:0:267915:0:(import.c:630:import_select_connection()) MGC192.168.202.148@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576206.657966:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.657970:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.657972:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.657975:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576206.657979:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576206.657982:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x14415860931531b3 00000020:00000001:0.0:1788576206.657985:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576206.657989:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c371eb37800 refcount=3 00000020:00000001:0.0:1788576206.657991:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000020:00000001:0.0:1788576206.657994:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359285053440 : -109714424498176 : ffff9c371eb37800) 00000100:00000040:0.0:1788576206.657997:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.657999:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.658001:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658003:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576206.658006:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c371eb37800 : new refcount 2 00000100:00080000:0.0:1788576206.658010:0:267915:0:(import.c:665:import_select_connection()) MGC192.168.202.148@tcp: import ffff9c373030d800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658014:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576206.658018:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576206.658022:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576206.658026:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.658029:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576206.658031:0:267915:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576206.658042:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788576206.658047:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c373030d800 refcount=4 obd=MGC192.168.202.148@tcp 02000000:00000001:0.0:1788576206.658052:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576206.658054:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.658057:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576206.658061:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576206.658067:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576206.658078:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20cc00. 00000100:00000001:0.0:1788576206.658095:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576206.658106:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810533888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576206.658117:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576206.658122:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.658127:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810533888/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576206.658544:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576206.658547:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576206.658551:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576206.658553:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576206.658554:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576206.658557:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576206.658561:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16673 00000400:00000200:0.0:1788576206.658565:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576206.658568:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576206.658571:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576206.658574:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658576:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.658578:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658580:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576206.658582:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576206.658584:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153207 00000020:00000001:0.0:1788576206.658585:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576206.658588:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c3720738800 refcount=5 00000020:00000001:0.0:1788576206.658590:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000020:00000001:0.0:1788576206.658593:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314417664 : -109714395133952 : ffff9c3720738800) 00000100:00000040:0.0:1788576206.658596:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658598:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.658607:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658608:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576206.658611:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c3720738800 : new refcount 4 00000100:00080000:0.0:1788576206.658613:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff9c3711155800: import ffff9c3707cce000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576206.658616:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576206.658618:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576206.658625:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576206.658627:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.658629:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576206.658993:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576206.658998:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659000:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576206.659005:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4e00. 00000020:00000040:0.0:1788576206.659008:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cce000 refcount=5 obd=lustre-MDT0000-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576206.659011:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576206.659012:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659015:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576206.659018:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576206.659021:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576206.659025:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20f000. 00000100:00000001:0.0:1788576206.659029:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576206.659034:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4e00 x1875440810534016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576206.659043:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576206.659046:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.659050:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfd78ff6] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4e00 x1875440810534016/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576206.659067:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576206.659070:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576206.659073:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576206.659074:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576206.659082:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576206.659084:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576206.659087:0:267915:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16673 00000400:00000200:0.0:1788576206.659091:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576206.659094:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576206.659097:0:267915:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576206.659100:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659102:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659104:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659105:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576206.659108:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576206.659109:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315320e 00000020:00000001:0.0:1788576206.659111:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576206.659114:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c372073a800 refcount=5 00000020:00000001:0.0:1788576206.659116:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000020:00000001:0.0:1788576206.659118:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634359314425856 : -109714395125760 : ffff9c372073a800) 00000100:00000040:0.0:1788576206.659121:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659123:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659124:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659126:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576206.659128:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c372073a800 : new refcount 4 00000100:00080000:0.0:1788576206.659131:0:267915:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff9c3711155800: import ffff9c3707ccf000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659133:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576206.659135:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576206.659138:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576206.659140:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659143:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576206.659146:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576206.659148:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659150:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576206.659157:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b4380. 00000020:00000040:0.0:1788576206.659159:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707ccf000 refcount=5 obd=lustre-MDT0001-mdc-ffff9c3711155800 02000000:00000001:0.0:1788576206.659162:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576206.659163:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659165:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576206.659166:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576206.659168:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576206.659172:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c400. 00000100:00000001:0.0:1788576206.659175:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576206.659179:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b4380 x1875440810534144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576206.659186:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576206.659188:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.659191:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045de8ad4] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b4380 x1875440810534144/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576206.659198:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576206.659201:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576206.659205:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576206.659206:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576206.659208:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576206.659210:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576206.659212:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16673 00000400:00000200:0.0:1788576206.659215:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576206.659218:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576206.659220:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576206.659222:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659223:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659225:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659227:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576206.659229:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576206.659234:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x144158609315321c 00000020:00000001:0.0:1788576206.659236:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576206.659238:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703a000 refcount=4 00000020:00000001:0.0:1788576206.659239:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000020:00000001:0.0:1788576206.659241:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887653376 : -109714821898240 : ffff9c370703a000) 00000100:00000040:0.0:1788576206.659245:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659247:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659249:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659251:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576206.659253:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703a000 : new refcount 3 00000100:00080000:0.0:1788576206.659256:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff9c3711155800: import ffff9c3707cca800 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659259:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576206.659260:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576206.659263:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576206.659264:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659266:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576206.659269:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576206.659271:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659273:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576206.659276:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5180. 00000020:00000040:0.0:1788576206.659278:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c3707cca800 refcount=7 obd=lustre-OST0000-osc-ffff9c3711155800 02000000:00000001:0.0:1788576206.659280:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576206.659281:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659283:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576206.659284:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576206.659286:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576206.659289:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20fc00. 00000100:00000001:0.0:1788576206.659293:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576206.659297:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5180 x1875440810534272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576206.659304:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576206.659306:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.659314:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000067c3b93] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5180 x1875440810534272/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576206.659322:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576206.659325:0:267915:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5fa795d2-07a0-4260-8bbc-bb3cde572406->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576206.659328:0:267915:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576206.659329:0:267915:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576206.659331:0:267915:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576206.659333:0:267915:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576206.659335:0:267915:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: connect to NID 192.168.202.148@tcp last attempt 16673 00000400:00000200:0.0:1788576206.659339:0:267915:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576206.659342:0:267915:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.148@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576206.659344:0:267915:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576206.659347:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659349:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659351:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659353:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000001:0.0:1788576206.659355:0:267915:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576206.659356:0:267915:0:(genops.c:804:class_conn2export()) looking for export cookie 0x1441586093153223 00000020:00000001:0.0:1788576206.659358:0:267915:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576206.659359:0:267915:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9c370703b800 refcount=4 00000020:00000001:0.0:1788576206.659362:0:267915:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000020:00000001:0.0:1788576206.659364:0:267915:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446634358887659520 : -109714821892096 : ffff9c370703b800) 00000100:00000040:0.0:1788576206.659367:0:267915:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff9c3703d9d100 refcount 14 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659368:0:267915:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576206.659370:0:267915:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff9c3703d9d100 refcount 15 to 192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659372:0:267915:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446634358834581760 : -109714874969856 : ffff9c3703d9d100) 00000020:00000040:0.0:1788576206.659374:0:267915:0:(genops.c:879:class_export_put()) PUTting export ffff9c370703b800 : new refcount 3 00000100:00080000:0.0:1788576206.659376:0:267915:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff9c3711155800: import ffff9c372073d000 using connection 192.168.202.148@tcp/192.168.202.148@tcp 00000100:00000001:0.0:1788576206.659378:0:267915:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576206.659386:0:267915:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576206.659389:0:267915:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576206.659391:0:267915:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659393:0:267915:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576206.659395:0:267915:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576206.659396:0:267915:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659398:0:267915:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576206.659410:0:267915:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c372d6b5500. 00000020:00000040:0.0:1788576206.659413:0:267915:0:(genops.c:1210:class_import_get()) import ffff9c372073d000 refcount=4 obd=lustre-OST0001-osc-ffff9c3711155800 02000000:00000001:0.0:1788576206.659415:0:267915:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576206.659416:0:267915:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576206.659418:0:267915:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576206.659419:0:267915:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576206.659421:0:267915:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576206.659425:0:267915:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c372d20c000. 00000100:00000001:0.0:1788576206.659428:0:267915:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576206.659433:0:267915:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c372d6b5500 x1875440810534400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576206.659440:0:267915:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576206.659442:0:267915:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.659445:0:267915:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055515cd2] to pc [ptlrpcd_rcv+-1] req@ffff9c372d6b5500 x1875440810534400/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576206.659452:0:267915:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576206.659457:0:267915:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16683) 00000100:00000040:0.0:1788576210.497739:0:268083:0:(connection.c:75:cpu_latency_work()) work item of ffff9c3712031800 (cpu 0) has reached its deadline 4311349471, at 4311349504 00000100:00000010:0.0:1788576210.497757:0:268083:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c3730391e40. 00000020:00000001:0.0:1788576211.203094:0:243062:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576211.203099:0:243062:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576211.203102:0:243062:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576211.780975:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576211.780983:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576211.780986:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576211.780997:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781030:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576211.781033:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.781037:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576211.781039:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576211.781041:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576211.781046:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810534528:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576211.781050:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576211.781054:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576211.781057:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.781059:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576211.781067:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.781069:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.781076:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576211.781081:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576211.781088:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576211.781095:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810534528, portal 25 00000100:00000001:0.0:1788576211.781097:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576211.781099:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291934720 : -109714417616896 : ffff9c371f1c7800) 00000100:00000040:0.0:1788576211.781105:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781112:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576211.781115:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576211.781118:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576211.781121:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440810534528, offset 0 00000400:00000200:0.0:1788576211.781127:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576211.781134:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.781142:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576211.781147:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576211.781151:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576211.781154:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576211.781156:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576211.781166:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544154:544154:256:4294967295] 192.168.202.148@tcp LPNI seq info [544119:544119:8:4294967295] 00000400:00000200:0.0:1788576211.781171:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.781180:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576211.781186:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.781191:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576211.781197:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576211.781201:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788576211.781213:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.781215:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576211.781220:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576211.781224:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576211.781229:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576211.781231:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576211.781235:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576211.781239:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781531:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.781537:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781544:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.781546:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576211.781553:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.781558:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576211.781561:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 0 is 4311354593 00000100:00000010:0.0:1788576211.781565:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626420. 00000100:00000001:0.0:1788576211.781571:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788576211.781579:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS new deadline estimation for cpu 1 is 4311354593 00000100:00000010:0.0:1788576211.781582:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c3711626180. 00000100:00000001:0.0:1788576211.781585:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.148@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788576211.781588:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576211.781596:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.781598:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576211.781632:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576211/real 1788576211] req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576211.781659:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576211.781661:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576211.781663:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576211.781666:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576211.781668:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576211.781670:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576211.781674:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576211.781682:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781688:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576211.781690:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781692:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.781694:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576211.781699:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781705:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576211.781709:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781715:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576211.781718:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781720:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.781721:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781723:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576211.781725:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576211.781728:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c373030d800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576211.781731:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576211.781740:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576211.781745:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576211.781753:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:874b6a47-e66d-4b10-8108-059b044e1170:2402:1875440810534528:192.168.202.148@tcp:250:kworker.0 00000100:00000001:0.0:1788576211.781758:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781760:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.781762:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7800 x1875440810534528/t0(0) o250->MGC192.168.202.148@tcp@192.168.202.148@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781769:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576211.781771:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576211.781774:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.781776:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576211.781778:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576211.781780:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c373030d800 refcount=3 obd=MGC192.168.202.148@tcp 00000020:00000001:0.0:1788576211.781782:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576211.781784:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233e400. 02000000:00000001:0.0:1788576211.781788:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576211.781790:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576211.781791:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7800. 00000100:00000001:0.0:1788576211.781794:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576211.781795:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781797:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.781804:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576211.781806:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.781808:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576211.781809:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576211.781811:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576211.781814:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781820:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576211.781822:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.781824:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576211.781825:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576211.781832:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576211.781835:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534656:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576211.781838:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576211.781840:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576211.781842:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.781843:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576211.781846:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.781847:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.781851:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576211.781854:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576211.781858:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576211.781860:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810534656, portal 10 00000100:00000001:0.0:1788576211.781862:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576211.781863:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291924864 : -109714417626752 : ffff9c371f1c5180) 00000100:00000040:0.0:1788576211.781869:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781875:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576211.781877:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576211.781878:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576211.781880:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810534656, offset 0 00000400:00000200:0.0:1788576211.781884:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576211.781888:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.781892:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576211.781894:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576211.781897:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576211.781899:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576211.781901:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576211.781904:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544155:544155:256:4294967295] 192.168.202.148@tcp LPNI seq info [544120:544120:8:4294967295] 00000400:00000200:0.0:1788576211.781908:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.781913:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576211.781917:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.781925:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576211.781930:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576211.781932:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788576211.781942:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.781944:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576211.781947:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576211.781950:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576211.781954:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576211.781955:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576211.781958:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576211.781961:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781968:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.781971:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.781977:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.781978:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576211.781981:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576211.781984:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: using a timeout of 23477 usecs (24 jiffies) 00000400:00000200:0.0:1788576211.781987:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576211.781988:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311350811 00000100:00000001:0.0:1788576211.781991:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.781992:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311350811 00000100:00000001:0.0:1788576211.781994:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.781996:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576211.781998:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.782000:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576211.782003:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576211/real 1788576211] req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576211.782025:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576211.782027:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576211.782028:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576211.782031:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576211.782032:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576211.782035:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576211.782038:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576211.782046:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.782053:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576211.782054:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782056:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.782057:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576211.782061:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.782067:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576211.782071:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.782078:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576211.782080:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782082:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.782083:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782085:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576211.782087:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576211.782089:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cce000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576211.782092:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576211.782094:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576211.782105:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576211.782112:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534656:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576211.782115:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782117:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.782119:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5180 x1875440810534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.782124:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576211.782126:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576211.782128:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.782130:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576211.782131:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576211.782132:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cce000 refcount=4 obd=lustre-MDT0000-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576211.782134:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576211.782135:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d800. 02000000:00000001:0.0:1788576211.782138:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576211.782139:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576211.782140:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5180. 00000100:00000001:0.0:1788576211.782142:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576211.782143:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782144:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.782148:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576211.782149:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.782151:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576211.782152:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.785197:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576211.785202:0:2402:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576211.785204:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576211.785211:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785219:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576211.785220:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785223:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576211.785240:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576211.785243:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576211.785247:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534784:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576211.785251:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576211.785255:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576211.785257:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785259:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576211.785263:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.785265:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.785270:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576211.785273:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576211.785279:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576211.785286:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810534784, portal 10 00000100:00000001:0.0:1788576211.785288:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576211.785290:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291925760 : -109714417625856 : ffff9c371f1c5500) 00000100:00000040:0.0:1788576211.785296:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785303:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576211.785306:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576211.785308:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576211.785311:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440810534784, offset 0 00000400:00000200:0.0:1788576211.785316:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576211.785323:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.785329:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576211.785333:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576211.785337:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576211.785340:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576211.785342:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576211.785346:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544156:544156:256:4294967295] 192.168.202.148@tcp LPNI seq info [544121:544121:8:4294967295] 00000400:00000200:0.0:1788576211.785350:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.785359:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576211.785365:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.785374:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576211.785381:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576211.785384:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788576211.785395:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.785397:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576211.785403:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576211.785407:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576211.785412:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576211.785414:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576211.785418:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576211.785422:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785431:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.785434:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785440:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.785441:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576211.785446:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576211.785449:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: using a timeout of 27271 usecs (28 jiffies) 00000400:00000200:0.0:1788576211.785453:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576211.785455:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311350819 00000100:00000001:0.0:1788576211.785458:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.785460:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311350819 00000100:00000001:0.0:1788576211.785462:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.785464:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576211.785466:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.785469:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576211.785473:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576211/real 1788576211] req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576211.785493:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576211.785495:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576211.785497:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576211.785500:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576211.785502:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576211.785504:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576211.785508:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576211.785515:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785521:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576211.785523:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785525:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.785527:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576211.785530:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785537:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576211.785542:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785548:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576211.785551:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785552:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.785553:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785556:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576211.785558:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576211.785561:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707ccf000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576211.785564:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576211.785567:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576211.785577:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576211.785585:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534784:192.168.202.148@tcp:38:kworker.0 00000100:00000001:0.0:1788576211.785589:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576211.785592:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785598:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576211.785599:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785601:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576211.785603:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576211.785616:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576211.785620:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534912:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576211.785623:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576211.785625:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576211.785626:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785628:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576211.785631:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576211.785633:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.785660:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576211.785664:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576211.785668:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576211.785671:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810534912, portal 4 00000100:00000001:0.0:1788576211.785673:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576211.785674:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291932928 : -109714417618688 : ffff9c371f1c7100) 00000100:00000040:0.0:1788576211.785680:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785686:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576211.785688:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576211.785691:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576211.785693:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810534912, offset 0 00000400:00000200:0.0:1788576211.785703:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576211.785707:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.785711:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576211.785713:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576211.785716:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576211.785718:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576211.785720:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576211.785723:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544157:544157:256:4294967295] 192.168.202.148@tcp LPNI seq info [544122:544122:8:4294967295] 00000400:00000200:0.0:1788576211.785727:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.785733:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576211.785737:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.785741:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576211.785745:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576211.785747:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788576211.785757:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.785759:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576211.785762:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576211.785765:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576211.785769:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576211.785771:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576211.785773:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576211.785777:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785784:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.785787:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785793:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.785794:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576211.785797:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576211.785800:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: using a timeout of 159995 usecs (160 jiffies) 00000400:00000200:0.0:1788576211.785803:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576211.785812:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311350952 00000100:00000001:0.0:1788576211.785815:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.785817:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311350952 00000100:00000001:0.0:1788576211.785819:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.785821:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576211.785823:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.785825:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576211.785830:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576211/real 1788576211] req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576211.785838:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576211.785839:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576211.785841:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576211.785844:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576211.785845:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576211.785847:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576211.785851:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576211.785859:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785865:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576211.785866:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785868:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.785869:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576211.785872:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785878:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576211.785882:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785888:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576211.785895:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785897:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.785898:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.785900:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576211.785902:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576211.785905:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c3707cca800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576211.785908:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576211.785910:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576211.785914:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576211.785921:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810534912:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576211.785925:0:2402:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788576211.785927:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785934:0:2402:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576211.785935:0:2402:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785937:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788576211.785939:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576211.785941:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576211.785943:0:2402:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810535040:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576211.785946:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788576211.785948:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788576211.785950:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576211.785952:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788576211.785955:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576211.785957:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788576211.785960:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000010:0.0:1788576211.785963:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c37109c8420. 00000400:00000010:0.0:1788576211.785966:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31c38. 00000100:00000200:0.0:1788576211.785969:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440810535040, portal 4 00000100:00000001:0.0:1788576211.785975:0:2402:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788576211.785976:0:2402:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446634359291921280 : -109714417630336 : ffff9c371f1c4380) 00000100:00000040:0.0:1788576211.785982:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.785988:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788576211.785990:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.148@tcp 00000400:00000010:0.0:1788576211.785992:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c372ed31330. 00000100:00000200:0.0:1788576211.785994:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440810535040, offset 0 00000400:00000200:0.0:1788576211.785998:0:2402:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.148@tcp 00000400:00000200:0.0:1788576211.786002:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.786006:0:2402:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.148@tcp local destination 00000400:00000200:0.0:1788576211.786009:0:2402:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788576211.786011:0:2402:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.48@tcp 00000400:00000200:0.0:1788576211.786014:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.48@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788576211.786016:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.148@tcp 00000400:00000200:0.0:1788576211.786019:0:2402:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.48@tcp NI seq info: [544158:544158:256:4294967295] 192.168.202.148@tcp LPNI seq info [544123:544123:8:4294967295] 00000400:00000200:0.0:1788576211.786023:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000400:00000200:0.0:1788576211.786028:0:2402:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.48@tcp(192.168.202.48@tcp:) -> 192.168.202.148@tcp(192.168.202.148@tcp:192.168.202.148@tcp) : PUT try# 0 00000800:00000200:0.0:1788576211.786033:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.786036:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c36010e0700. 00000800:00000200:0.0:1788576211.786040:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c3710be4c00] -> 12345-192.168.202.148@tcp (2) 00000800:00000200:0.0:1788576211.786042:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.148 (cnted 0, interval 60, 30 secs later) 00000800:00000100:0.0:1788576211.786052:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.148@tcp 00000800:00000010:0.0:1788576211.786054:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c36010e0700. 00000400:00000200:0.0:1788576211.786056:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788576211.786060:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.48@tcp->192.168.202.148@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788576211.786063:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31330 00000400:00000010:0.0:1788576211.786064:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31330. 00000100:00000001:0.0:1788576211.786067:0:2402:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788576211.786070:0:2402:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786089:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.786093:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786099:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.786100:0:2402:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788576211.786105:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576211.786108:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: using a timeout of 168830 usecs (169 jiffies) 00000400:00000200:0.0:1788576211.786111:0:2402:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.148@tcp to cpt 0 00000100:00000001:0.0:1788576211.786113:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 0 is 4311350961 00000100:00000001:0.0:1788576211.786115:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 0, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.786117:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS new deadline estimation for cpu 1 is 4311350961 00000100:00000001:0.0:1788576211.786119:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c3711155800: PM QoS request active for cpu 1, keep current deadline 4311354593 00000100:00000001:0.0:1788576211.786121:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788576211.786124:0:2402:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.786126:0:2402:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788576211.786130:0:2402:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576211/real 1788576211] req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788576211.786137:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788576211.786138:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788576211.786140:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c37109c8420. 00000400:00000200:0.0:1788576211.786142:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c372ed31c38 00000400:00000010:0.0:1788576211.786144:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9c372ed31c38. 00000100:00000001:0.0:1788576211.786146:0:2402:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788576211.786149:0:2402:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788576211.786156:0:2402:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786162:0:2402:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788576211.786169:0:2402:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786170:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.786171:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788576211.786175:0:2402:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786181:0:2402:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788576211.786185:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786192:0:2402:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788576211.786194:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786195:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788576211.786197:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786199:0:2402:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788576211.786200:0:2402:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788576211.786203:0:2402:0:(import.c:65:import_set_state_nolock()) ffff9c372073d000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788576211.786206:0:2402:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.148@tcp failed (-110) 00000100:00000001:0.0:1788576211.786208:0:2402:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788576211.786212:0:2402:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788576211.786219:0:2402:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff9c371f1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5fa795d2-07a0-4260-8bbc-bb3cde572406:2402:1875440810535040:192.168.202.148@tcp:8:kworker.0 00000100:00000001:0.0:1788576211.786222:0:2402:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786224:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.786227:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c5500 x1875440810534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c3711155800@192.168.202.148@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786234:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576211.786236:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576211.786239:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20d800. 02000000:00000001:0.0:1788576211.786241:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576211.786243:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576211.786248:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707ccf000 refcount=4 obd=lustre-MDT0001-mdc-ffff9c3711155800 00000020:00000001:0.0:1788576211.786251:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576211.786254:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233f800. 02000000:00000001:0.0:1788576211.786257:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576211.786259:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576211.786260:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c5500. 00000100:00000001:0.0:1788576211.786264:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576211.786265:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786267:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.786270:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c7100 x1875440810534912/t0(0) o8->lustre-OST0000-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786276:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576211.786277:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576211.786279:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20c000. 02000000:00000001:0.0:1788576211.786281:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576211.786282:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576211.786283:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c3707cca800 refcount=6 obd=lustre-OST0000-osc-ffff9c3711155800 00000020:00000001:0.0:1788576211.786285:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576211.786286:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233d400. 02000000:00000001:0.0:1788576211.786288:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576211.786289:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576211.786290:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c7100. 00000100:00000001:0.0:1788576211.786293:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576211.786294:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786295:0:2402:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788576211.786298:0:2402:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c371f1c4380 x1875440810535040/t0(0) o8->lustre-OST0001-osc-ffff9c3711155800@192.168.202.148@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576211.786305:0:2402:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788576211.786306:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788576211.786307:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c372d20fc00. 02000000:00000001:0.0:1788576211.786309:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788576211.786310:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788576211.786311:0:2402:0:(genops.c:1223:class_import_put()) import ffff9c372073d000 refcount=3 obd=lustre-OST0001-osc-ffff9c3711155800 00000020:00000001:0.0:1788576211.786313:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788576211.786314:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c373233fc00. 02000000:00000001:0.0:1788576211.786316:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788576211.786323:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788576211.786324:0:2402:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c371f1c4380. 00000100:00000001:0.0:1788576211.786327:0:2402:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788576211.786328:0:2402:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786330:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576211.786336:0:2402:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576211.786338:0:2402:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576211.786340:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788576211.786342:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) Debug log: 139203 lines, 139203 kept, 0 dropped, 0 bad.